diff --git a/src/backend/datasources/FileDataSource.cpp b/src/backend/datasources/FileDataSource.cpp index 69d03a61a..4860f6e71 100644 --- a/src/backend/datasources/FileDataSource.cpp +++ b/src/backend/datasources/FileDataSource.cpp @@ -1,984 +1,992 @@ /*************************************************************************** File : FileDataSource.cpp Project : LabPlot Description : Represents file data source -------------------------------------------------------------------- Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Fabian Kristof (fkristofszabolcs@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/FileDataSource.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/FITSFilter.h" #include "backend/datasources/filters/BinaryFilter.h" #include "backend/core/Project.h" #include "kdefrontend/spreadsheet/PlotDataDialog.h" #include "commonfrontend/spreadsheet/SpreadsheetView.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*! \class FileDataSource \brief Represents data stored in a file. Reading and writing is done with the help of appropriate I/O-filters. \ingroup datasources */ FileDataSource::FileDataSource(AbstractScriptingEngine* engine, const QString& name, bool loading) : Spreadsheet(engine, name, loading), m_fileType(Ascii), m_fileWatched(false), m_fileLinked(false), m_paused(false), m_prepared(false), m_keepLastValues(false), m_bytesRead(0), m_filter(nullptr), m_updateTimer(new QTimer(this)), m_fileSystemWatcher(nullptr), m_file(nullptr), m_localSocket(nullptr), m_tcpSocket(nullptr), m_serialPort(nullptr), m_device(nullptr) { initActions(); connect(m_updateTimer, SIGNAL(timeout()), this, SLOT(read())); } FileDataSource::~FileDataSource() { if (m_filter) delete m_filter; if (m_fileSystemWatcher) delete m_fileSystemWatcher; if (m_file) delete m_file; if (m_localSocket) delete m_localSocket; if (m_tcpSocket) delete m_tcpSocket; if (m_serialPort) delete m_serialPort; delete m_updateTimer; } /*! * depending on the update type, periodically or on data changes, starts the timer or activates the file watchers, respectively. */ void FileDataSource::ready() { if (m_updateType == TimeInterval) m_updateTimer->start(m_updateInterval); else watch(); } void FileDataSource::initActions() { m_reloadAction = new QAction(QIcon::fromTheme("view-refresh"), i18n("Reload"), this); connect(m_reloadAction, SIGNAL(triggered()), this, SLOT(read())); m_toggleLinkAction = new QAction(i18n("Link the file"), this); m_toggleLinkAction->setCheckable(true); connect(m_toggleLinkAction, SIGNAL(triggered()), this, SLOT(linkToggled())); m_plotDataAction = new QAction(QIcon::fromTheme("office-chart-line"), i18n("Plot data"), this); connect(m_plotDataAction, SIGNAL(triggered()), this, SLOT(plotData())); } //TODO make the view customizable (show as a spreadsheet or as a pure text file in an editor) QWidget *FileDataSource::view() const { if (!m_view) m_view = new SpreadsheetView(const_cast(this)); return m_view; } /*! * \brief Returns a list with the names of the available ports */ QStringList FileDataSource::availablePorts() { QStringList ports; qDebug() << "available ports count:" << QSerialPortInfo::availablePorts().size(); for(const QSerialPortInfo& sp : QSerialPortInfo::availablePorts()) { ports.append(sp.portName()); qDebug() << sp.description(); qDebug() << sp.manufacturer(); qDebug() << sp.portName(); qDebug() << sp.serialNumber(); qDebug() << sp.systemLocation(); } return ports; } /*! * \brief Returns a list with the supported baud rates */ QStringList FileDataSource::supportedBaudRates() { QStringList baudRates; for(const auto& baud : QSerialPortInfo::standardBaudRates()) baudRates.append(QString::number(baud)); return baudRates; } /*! * \brief Updates this data source at this moment */ void FileDataSource::updateNow() { m_updateTimer->stop(); read(); //restart the timer after update if (m_updateType == TimeInterval) m_updateTimer->start(m_updateInterval); } /*! * \brief FileDataSource::stopReading */ //TODO: do we want this? void FileDataSource::stopReading() { if (m_updateType == TimeInterval) m_updateTimer->stop(); else if (m_updateType == NewData) disconnect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(read())); } /*! * \brief Continue reading from the live data source after it was paused. */ void FileDataSource::continueReading() { m_paused = false; if (m_updateType == TimeInterval) m_updateTimer->start(m_updateInterval); else if (m_updateType == NewData) connect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(read())); } /*! * \brief Pause the reading of the live data source. */ void FileDataSource::pauseReading() { m_paused = true; if (m_updateType == TimeInterval) m_updateTimer->stop(); else if (m_updateType == NewData) disconnect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(read())); } /*! returns the list with all supported data file formats. */ QStringList FileDataSource::fileTypes() { // see FileDataSource::FileType return (QStringList()<< i18n("ASCII data") << i18n("Binary data") << i18n("Image") << i18n("Hierarchical Data Format (HDF)") << i18n("Network Common Data Format (NetCDF)") // << "CDF" << i18n("Flexible Image Transport System Data Format (FITS)") // << i18n("Sound") ); } void FileDataSource::setFileName(const QString& name) { m_fileName = name; } QString FileDataSource::fileName() const { return m_fileName; } /*! * \brief Sets the local socket's server name to name * \param name */ void FileDataSource::setLocalSocketName(const QString & name) { m_localSocketName = name; } QString FileDataSource::localSocketName() const { return m_localSocketName; } void FileDataSource::setFileType(const FileType type) { m_fileType = type; } FileDataSource::FileType FileDataSource::fileType() const { return m_fileType; } void FileDataSource::setFilter(AbstractFileFilter* f) { m_filter = f; } AbstractFileFilter* FileDataSource::filter() const { return m_filter; } /*! sets whether the file should be watched or not. In the first case the data source will be automatically updated on file changes. */ void FileDataSource::setFileWatched(const bool b) { m_fileWatched = b; } bool FileDataSource::isFileWatched() const { return m_fileWatched; } /*! * \brief Sets whether we'll keep the last values or append it to the previous ones * \param keepLastValues */ void FileDataSource::setKeepLastValues(const bool keepLastValues) { m_keepLastValues = keepLastValues; } bool FileDataSource::keepLastValues() const { return m_keepLastValues; } /*! * \brief Sets the serial port's baud rate * \param baudrate */ void FileDataSource::setBaudRate(const int baudrate) { m_baudRate = baudrate; } int FileDataSource::baudRate() const { return m_baudRate; } /*! * \brief Sets the source's update interval to \c interval * \param interval */ void FileDataSource::setUpdateInterval(const int interval) { m_updateInterval = interval; m_updateTimer->start(m_updateInterval); } int FileDataSource::updateInterval() const { return m_updateInterval; } /*! * \brief Sets how many values we should store * \param keepnvalues */ void FileDataSource::setKeepNvalues(const int keepnvalues) { m_keepNvalues = keepnvalues; } int FileDataSource::keepNvalues() const { return m_keepNvalues; } /*! * \brief Sets the network socket's port to port * \param port */ void FileDataSource::setPort(const int port) { m_port = port; } +void FileDataSource::setBytesRead(const qint64 bytes) { + m_bytesRead = bytes; +} + +int FileDataSource::bytesRead() const { + return m_bytesRead; +} + int FileDataSource::port() const { return m_port; } /*! * \brief Sets the serial port's name to name * \param name */ void FileDataSource::setSerialPort(const QString &name) { m_serialPortName = name; } QString FileDataSource::serialPortName() const { return m_serialPortName; } /*! * \brief Sets the sample rate to samplerate * \param samplerate */ void FileDataSource::setSampleRate(const int samplerate) { m_sampleRate = samplerate; } int FileDataSource::sampleRate() const { return m_sampleRate; } /*! * \brief Sets the source's type to sourcetype * \param sourcetype */ void FileDataSource::setSourceType(const SourceType sourcetype) { m_sourceType = sourcetype; } FileDataSource::SourceType FileDataSource::sourceType() const { return m_sourceType; } /*! * \brief Sets the source's reading type to readingType * \param readingType */ void FileDataSource::setReadingType(const ReadingType readingType) { m_readingType = readingType; } FileDataSource::ReadingType FileDataSource::readingType() const { return m_readingType; } /*! * \brief Sets the source's update type to updatetype and handles this change * \param updatetype */ void FileDataSource::setUpdateType(const UpdateType updatetype) { if (updatetype == NewData) { m_updateTimer->stop(); if (m_fileSystemWatcher == nullptr) watch(); else connect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(read())); } else disconnect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(read())); m_updateType = updatetype; } FileDataSource::UpdateType FileDataSource::updateType() const { return m_updateType; } /*! * \brief Sets the network socket's host * \param host */ void FileDataSource::setHost(const QString & host) { m_host = host; } QString FileDataSource::host() const { return m_host; } /*! sets whether only a link to the file is saved in the project file (\c b=true) or the whole content of the file (\c b=false). */ void FileDataSource::setFileLinked(const bool b) { m_fileLinked = b; } /*! returns \c true if only a link to the file is saved in the project file. \c false otherwise. */ bool FileDataSource::isFileLinked() const { return m_fileLinked; } QIcon FileDataSource::icon() const { QIcon icon; if (m_fileType == FileDataSource::Ascii) icon = QIcon::fromTheme("text-plain"); else if (m_fileType == FileDataSource::Binary) icon = QIcon::fromTheme("application-octet-stream"); else if (m_fileType == FileDataSource::Image) icon = QIcon::fromTheme("image-x-generic"); // TODO: HDF, NetCDF, FITS, etc. return icon; } QMenu* FileDataSource::createContextMenu() { QMenu* menu = AbstractPart::createContextMenu(); QAction* firstAction = 0; // if we're populating the context menu for the project explorer, then //there're already actions available there. Skip the first title-action //and insert the action at the beginning of the menu. if (menu->actions().size()>1) firstAction = menu->actions().at(1); menu->insertAction(firstAction, m_plotDataAction); menu->insertSeparator(firstAction); //TODO: doesnt' always make sense... // if (!m_fileWatched) // menu->insertAction(firstAction, m_reloadAction); // // m_toggleWatchAction->setChecked(m_fileWatched); // menu->insertAction(firstAction, m_toggleWatchAction); // // m_toggleLinkAction->setChecked(m_fileLinked); // menu->insertAction(firstAction, m_toggleLinkAction); return menu; } //############################################################################## //################################# SLOTS #################################### //############################################################################## /* * called periodically or on new data changes (file changed, new data in the socket, etc.) */ void FileDataSource::read() { if (m_filter == nullptr) return; //initialize the device (file, socket, serial port), when calling this function for the first time if (!m_prepared) { switch (m_sourceType) { case FileOrPipe: m_file = new QFile(m_fileName); m_device = m_file; break; case NetworkSocket: m_tcpSocket = new QTcpSocket; m_device = m_tcpSocket; connect(m_tcpSocket, SIGNAL(readyRead()), this, SLOT(readyRead())); connect(m_tcpSocket, SIGNAL(error(QAbstractSocket::SocketError)), this, SLOT(tcpSocketError(QAbstractSocket::SocketError))); break; case LocalSocket: m_localSocket = new QLocalSocket(this); qDebug() << "socket state before preparing: " << m_localSocket->state(); m_localSocket->connectToServer(m_localSocketName, QLocalSocket::ReadOnly); qDebug() << "socket state after preparing: " << m_localSocket->state(); m_device = m_localSocket; connect(m_localSocket, SIGNAL(readyRead()), this, SLOT(readyRead())); connect(m_localSocket, SIGNAL(error(QLocalSocket::LocalSocketError)), this, SLOT(localSocketError(QLocalSocket::LocalSocketError))); break; case SerialPort: m_serialPort = new QSerialPort; m_device = m_serialPort; m_serialPort->setBaudRate(m_baudRate); m_serialPort->setPortName(m_serialPortName); connect(m_serialPort, SIGNAL(error(QSerialPort::SerialPortError)), this, SLOT(serialPortError(QSerialPort::SerialPortError))); connect(m_serialPort, SIGNAL(readyRead()), this, SLOT(readyRead())); break; } m_prepared = true; } qint64 bytes = 0; switch (m_sourceType) { case FileOrPipe: switch (m_fileType) { case Ascii: qDebug() << "Reading live ascii file.." ; bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_file, this, m_bytesRead); m_bytesRead += bytes; qDebug() << "Read " << bytes << " bytes, in total: " << m_bytesRead; break; case Binary: //bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_file, this, m_bytesRead); m_bytesRead += bytes; default: break; } break; case NetworkSocket: DEBUG("reading from a network socket"); m_tcpSocket->abort(); m_tcpSocket->connectToHost(m_host, m_port, QIODevice::ReadOnly); break; case LocalSocket: DEBUG("reading from a local socket"); qDebug() << m_localSocket->state(); m_localSocket->abort(); m_localSocket->connectToServer(m_localSocketName, QLocalSocket::ReadOnly); qDebug() << m_localSocket->state(); break; case SerialPort: DEBUG("reading from the serial port"); //TODO break; } } /*! * Slot for the signal that is emitted once every time new data is available for reading from the device. * It will only be emitted again once new data is available, such as when a new payload of network data has arrived on the network socket, * or when a new block of data has been appended to your device. */ void FileDataSource::readyRead() { DEBUG("Got new data from the device"); if (m_fileType == Ascii) dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_device, this); // else if (m_fileType == Binary) // dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_device, this); //since we won't have the timer to call read() where we create new connections //for sequencial devices in read() we just request data/connect to servers if (m_updateType == NewData) read(); } void FileDataSource::localSocketError(QLocalSocket::LocalSocketError socketError) { /*disconnect(m_localSocket, SIGNAL(error(QLocalSocket::LocalSocketError)), this, SLOT(localSocketError(QLocalSocket::LocalSocketError))); disconnect(m_localSocket, SIGNAL(readyRead()), this, SLOT(readyRead()));*/ switch (socketError) { case QLocalSocket::ServerNotFoundError: QMessageBox::critical(0, i18n("Local Socket Error"), i18n("The socket was not found. Please check the socket name.")); break; case QLocalSocket::ConnectionRefusedError: QMessageBox::critical(0, i18n("Local Socket Error"), i18n("The connection was refused by the peer")); break; case QLocalSocket::PeerClosedError: QMessageBox::critical(0, i18n("Local Socket Error"), i18n("The socket has closed the connection.")); break; default: QMessageBox::critical(0, i18n("Local Socket Error"), i18n("The following error occurred: %1.").arg(m_localSocket->errorString())); } } void FileDataSource::tcpSocketError(QAbstractSocket::SocketError socketError) { switch (socketError) { case QAbstractSocket::ConnectionRefusedError: QMessageBox::critical(0, i18n("TCP Socket Error"), i18n("The connection was refused by the peer. Make sure the server is running and check the host name and port settings.")); break; case QAbstractSocket::RemoteHostClosedError: QMessageBox::critical(0, i18n("TCP Socket Error"), i18n("The remote host closed the connection.")); break; case QAbstractSocket::HostNotFoundError: QMessageBox::critical(0, i18n("TCP Socket Error"), i18n("The host was not found. Please check the host name and port settings.")); break; default: QMessageBox::critical(0, i18n("TCP Socket Error"), i18n("The following error occurred: %1.").arg(m_tcpSocket->errorString())); } } void FileDataSource::serialPortError(QSerialPort::SerialPortError serialPortError) { switch (serialPortError) { case QSerialPort::DeviceNotFoundError: QMessageBox::critical(0, i18n("Serial Port Error"), i18n("Failed to open the device.")); break; case QSerialPort::PermissionError: QMessageBox::critical(0, i18n("Serial Port Error"), i18n("Failed to open the device. Please check your permissions on this device.")); break; case QSerialPort::OpenError: QMessageBox::critical(0, i18n("Serial Port Error"), i18n("Device already opened.")); break; case QSerialPort::NotOpenError: QMessageBox::critical(0, i18n("Serial Port Error"), i18n("The device is not opened.")); break; case QSerialPort::ReadError: QMessageBox::critical(0, i18n("Serial Port Error"), i18n("Failed to read data.")); break; case QSerialPort::ResourceError: QMessageBox::critical(0, i18n("Serial Port Error"), i18n("Failed to read data. The device is removed.")); break; case QSerialPort::TimeoutError: QMessageBox::critical(0, i18n("Serial Port Error"), i18n("The device timed out.")); break; default: QMessageBox::critical(0, i18n("Serial Port Error"), i18n("The following error occurred: %1.").arg(m_serialPort->errorString())); break; } } void FileDataSource::watchToggled() { m_fileWatched = !m_fileWatched; watch(); project()->setChanged(true); } void FileDataSource::linkToggled() { m_fileLinked = !m_fileLinked; project()->setChanged(true); } //watch the file upon reading for changes if required void FileDataSource::watch() { if (m_fileWatched) { if (!m_fileSystemWatcher) { m_fileSystemWatcher = new QFileSystemWatcher; connect (m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(read())); } if ( !m_fileSystemWatcher->files().contains(m_fileName) ) m_fileSystemWatcher->addPath(m_fileName); } else { if (m_fileSystemWatcher) m_fileSystemWatcher->removePath(m_fileName); } } /*! returns a string containing the general information about the file \c name and some content specific information (number of columns and lines for ASCII, color-depth for images etc.). */ QString FileDataSource::fileInfoString(const QString &name) { QString infoString; QFileInfo fileInfo; QString fileTypeString; QIODevice *file = new QFile(name); QString fileName; if (name.at(0) != QDir::separator()) fileName = QDir::homePath() + QDir::separator() + name; else fileName = name; if(file==0) file = new QFile(fileName); if (file->open(QIODevice::ReadOnly)) { QStringList infoStrings; //general information about the file infoStrings << "" + fileName + "
"; fileInfo.setFile(fileName); infoStrings << i18n("Readable: %1", fileInfo.isReadable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Writable: %1", fileInfo.isWritable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Executable: %1", fileInfo.isExecutable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Created: %1", fileInfo.created().toString()); infoStrings << i18n("Last modified: %1", fileInfo.lastModified().toString()); infoStrings << i18n("Last read: %1", fileInfo.lastRead().toString()); infoStrings << i18n("Owner: %1", fileInfo.owner()); infoStrings << i18n("Group: %1", fileInfo.group()); infoStrings << i18n("Size: %1", i18np("%1 cByte", "%1 cBytes", fileInfo.size())); #ifdef HAVE_FITS if (fileName.endsWith(QLatin1String(".fits"))) { FITSFilter* fitsFilter = new FITSFilter; infoStrings << i18n("Images: %1", QString::number(fitsFilter->imagesCount(fileName) )); infoStrings << i18n("Tables: %1", QString::number(fitsFilter->tablesCount(fileName) )); delete fitsFilter; } #endif // file type and type specific information about the file #ifdef Q_OS_LINUX QProcess *proc = new QProcess(); QStringList args; args<<"-b"<start( "file", args); if(proc->waitForReadyRead(1000) == false) infoStrings << i18n("Could not open file %1 for reading.", fileName); else { fileTypeString = proc->readLine(); if( fileTypeString.contains(i18n("cannot open")) ) fileTypeString=""; else { fileTypeString.remove(fileTypeString.length()-1,1); // remove '\n' } } infoStrings << i18n("File type: %1", fileTypeString); #endif //TODO depending on the file type, generate additional information about the file: //Number of lines for ASCII, color-depth for images etc. Use the specific filters here. // port the old labplot1.6 code. if( fileTypeString.contains("ASCII")) { infoStrings << "
"; //TODO: consider choosen separator infoStrings << i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName)); infoStrings << i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); } infoString += infoStrings.join("
"); } else infoString += i18n("Could not open file %1 for reading.", fileName); return infoString; } void FileDataSource::plotData() { PlotDataDialog* dlg = new PlotDataDialog(this); dlg->exec(); } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void FileDataSource::save(QXmlStreamWriter* writer) const { writer->writeStartElement("fileDataSource"); writeBasicAttributes(writer); writeCommentElement(writer); //general writer->writeStartElement("general"); writer->writeAttribute("fileName", m_fileName); writer->writeAttribute("fileType", QString::number(m_fileType)); writer->writeAttribute("fileWatched", QString::number(m_fileWatched)); writer->writeAttribute("fileLinked", QString::number(m_fileLinked)); writer->writeAttribute("updateType", QString::number(m_updateType)); writer->writeAttribute("readingType", QString::number(m_readingType)); writer->writeAttribute("sourceType", QString::number(m_sourceType)); writer->writeAttribute("keepValues", QString::number(m_keepNvalues)); if (m_updateType == TimeInterval) writer->writeAttribute("updateInterval", QString::number(m_updateInterval)); if (m_readingType != TillEnd) writer->writeAttribute("sampleRate", QString::number(m_sampleRate)); switch (m_sourceType) { case SerialPort: writer->writeAttribute("baudRate", QString::number(m_baudRate)); writer->writeAttribute("serialPortName", m_serialPortName); break; case NetworkSocket: writer->writeAttribute("host", m_host); writer->writeAttribute("port", QString::number(m_port)); break; case FileOrPipe: break; case LocalSocket: break; default: break; } writer->writeEndElement(); //filter m_filter->save(writer); //columns if (!m_fileLinked) { foreach (Column * col, children(IncludeHidden)) col->save(writer); } writer->writeEndElement(); // "fileDataSource" } /*! Loads from XML. */ bool FileDataSource::load(XmlStreamReader* reader) { if(!reader->isStartElement() || reader->name() != "fileDataSource") { reader->raiseError(i18n("no fileDataSource element found")); return false; } if (!readBasicAttributes(reader)) return false; QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs; QString str; while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement() && reader->name() == "fileDataSource") break; if (!reader->isStartElement()) continue; if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if (reader->name() == "general") { attribs = reader->attributes(); str = attribs.value("fileName").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileName'")); else m_fileName = str; str = attribs.value("fileType").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileType'")); else m_fileType = (FileType)str.toInt(); str = attribs.value("fileWatched").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileWatched'")); else m_fileWatched = str.toInt(); str = attribs.value("fileLinked").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileLinked'")); else m_fileLinked = str.toInt(); str = attribs.value("updateType").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'updateType'")); else m_updateType = static_cast(str.toInt()); str = attribs.value("sourceType").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'sourceType'")); else m_sourceType = static_cast(str.toInt()); str = attribs.value("readingType").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'readingType'")); else m_readingType = static_cast(str.toInt()); if (m_updateType == TimeInterval) { str = attribs.value("updateInterval").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'updateInterval'")); else m_updateInterval = str.toInt(); } if (m_readingType != TillEnd) { str = attribs.value("sampleRate").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'sampleRate'")); else m_sampleRate = str.toInt(); } switch (m_sourceType) { case SerialPort: str = attribs.value("baudRate").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'baudRate'")); else m_baudRate = str.toInt(); str = attribs.value("serialPortName").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'serialPortName'")); else m_serialPortName = str; break; case NetworkSocket: str = attribs.value("host").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'host'")); else m_host = str; str = attribs.value("port").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'port'")); else m_host = str; break; case FileOrPipe: break; case LocalSocket: break; default: break; } } else if (reader->name() == "asciiFilter") { m_filter = new AsciiFilter(); if (!m_filter->load(reader)) return false; } else if(reader->name() == "column") { Column* column = new Column("", AbstractColumn::Text); if (!column->load(reader)) { delete column; setColumnCount(0); return false; } addChild(column); } else {// unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } //read the content of the file if it was only linked if (m_fileLinked) this->read(); return !reader->hasError(); } diff --git a/src/backend/datasources/FileDataSource.h b/src/backend/datasources/FileDataSource.h index 943c54c18..b1352a1ae 100644 --- a/src/backend/datasources/FileDataSource.h +++ b/src/backend/datasources/FileDataSource.h @@ -1,204 +1,207 @@ /*************************************************************************** File : FileDataSource.h Project : LabPlot Description : File data source -------------------------------------------------------------------- Copyright : (C) 2012-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Fabian Kristof (fkristofszabolcs@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef FILEDATASOURCE_H #define FILEDATASOURCE_H #include "backend/spreadsheet/Spreadsheet.h" #include "backend/matrix/Matrix.h" #include #include #include class QString; class AbstractFileFilter; class QFileSystemWatcher; class QAction; class QTcpSocket; class QFile; class FileDataSource : public Spreadsheet { Q_OBJECT Q_ENUMS(FileType) public: enum FileType {Ascii, Binary, Image, HDF, NETCDF, FITS}; enum SourceType { FileOrPipe = 0, NetworkSocket, LocalSocket, SerialPort }; enum UpdateType { TimeInterval = 0, NewData }; enum ReadingType { ContinousFixed = 0, FromEnd, TillEnd }; FileDataSource(AbstractScriptingEngine*, const QString& name, bool loading = false); ~FileDataSource(); void ready(); static QStringList supportedBaudRates(); static QStringList availablePorts(); static QStringList fileTypes(); static QString fileInfoString(const QString&); void setFileType(const FileType); FileType fileType() const; UpdateType updateType() const; void setUpdateType(const UpdateType); SourceType sourceType() const; void setSourceType(const SourceType); ReadingType readingType() const; void setReadingType(const ReadingType); int sampleRate() const; void setSampleRate(const int); + void setBytesRead(const qint64 bytes); + int bytesRead() const; + int port() const; void setPort(const int); void setSerialPort(const QString& name); QString serialPortName() const; QString host() const; void setHost(const QString&); int baudRate() const; void setBaudRate(const int); void setUpdateInterval(const int); int updateInterval() const; void setKeepNvalues(const int); int keepNvalues() const; void setKeepLastValues(const bool); bool keepLastValues() const; void setFileWatched(const bool); bool isFileWatched() const; void setFileLinked(const bool); bool isFileLinked() const; void setFileName(const QString&); QString fileName() const; void setLocalSocketName(const QString&); QString localSocketName() const; void updateNow(); void stopReading(); void pauseReading(); void continueReading(); void setFilter(AbstractFileFilter*); AbstractFileFilter* filter() const; virtual QIcon icon() const; virtual QMenu* createContextMenu(); virtual QWidget* view() const; virtual void save(QXmlStreamWriter*) const; virtual bool load(XmlStreamReader*); private: void initActions(); void watch(); QString m_fileName; QString m_serialPortName; QString m_localSocketName; QString m_host; FileType m_fileType; UpdateType m_updateType; SourceType m_sourceType; ReadingType m_readingType; bool m_fileWatched; bool m_fileLinked; bool m_paused; bool m_prepared; bool m_keepLastValues; int m_sampleRate; int m_keepNvalues; int m_updateInterval; int m_port; int m_baudRate; qint64 m_bytesRead; AbstractFileFilter* m_filter; QTimer* m_updateTimer; QFileSystemWatcher* m_fileSystemWatcher; QFile* m_file; QLocalSocket* m_localSocket; QTcpSocket* m_tcpSocket; QSerialPort* m_serialPort; QIODevice* m_device; QAction* m_reloadAction; QAction* m_toggleLinkAction; QAction* m_showEditorAction; QAction* m_showSpreadsheetAction; QAction* m_plotDataAction; public slots: void read(); private slots: void watchToggled(); void linkToggled(); void plotData(); void readyRead(); void localSocketError(QLocalSocket::LocalSocketError); void tcpSocketError(QAbstractSocket::SocketError); void serialPortError(QSerialPort::SerialPortError); }; #endif diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index d58792a81..5d7880bab 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,1139 +1,1142 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2017 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/FileDataSource.h" #include "backend/core/column/Column.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/lib/macros.h" #include #include #include #include #include #include /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() {} /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource * dataSource, AbstractFileFilter::ImportMode) { d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from, AbstractFileFilter::ImportMode importMode) { return d->readFromLiveDevice(device, dataSource, from, importMode); } /*! reads the content of the file \c fileName. */ QVector AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromFile(fileName, dataSource, importMode, lines); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; i++) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of lines"); return 0; } size_t lineCount = 0; while (!device.atEnd()) { device.readLine(); lineCount++; } //TODO: wc is much faster but not portable /* QElapsedTimer myTimer; myTimer.start(); QProcess wc; wc.start(QString("wc"), QStringList() << "-l" << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) lineCount = wc.readLine().split(' ')[0].toInt(); lineCount++; // last line not counted DEBUG(" Elapsed time counting lines : " << myTimer.elapsed() << " ms"); */ return lineCount; } /*! returns the number of lines in the device \c device or 0 if not available. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice &device) { // device.hasReadLine() always returns 0 for KFilterDev! if (device.isSequential()) return 0; size_t lineCount = 0; device.seek(0); while (!device.atEnd()) { device.readLine(); lineCount++; } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setVectorNames(const QString s) { d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner), commentCharacter("#"), separatingCharacter("auto"), autoModeEnabled(true), headerEnabled(true), skipEmptyParts(false), simplifyWhitespacesEnabled(true), createIndexEnabled(false), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_prepared(false), m_columnOffset(0) { } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd()) // empty file return 1; // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines firstLine = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); QString firstLineOriginal = firstLine; firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, QString::SkipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, QString::SkipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); DEBUG("headerEnabled = " << headerEnabled); if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); startRow++; } // set range to read if (endColumn == -1) endColumn = firstLineStringList.size(); // last column if (createIndexEnabled) { vectorNames.prepend("index"); endColumn++; } m_actualCols = endColumn - startColumn + 1; //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ // this also resets position to start of file m_actualRows = AsciiFilter::lineNumber(device); // Find first data line (ignoring comment lines) DEBUG("Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; ++i) { QString line = device.readLine(); if (device.atEnd()) { if (device.isSequential()) break; else return 1; } if (line.startsWith(commentCharacter)) // ignore commented lines i--; } // parse first data line to determine data type for each column if (device.isSequential()) firstLine = firstLineOriginal; else firstLine = device.readLine(); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("first data line : \'" << firstLine.toStdString() << '\''); firstLineStringList = firstLine.split(m_separator, QString::SkipEmptyParts); QDEBUG("first data line, parsed: " << firstLineStringList); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Numeric; col = 1; } for (const auto& valueString: firstLineStringList) { // only parse columns available in first data line if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } QDEBUG("column modes = " << columnModes); int actualEndRow = endRow; DEBUG("endRow = " << endRow); if (endRow == -1 || endRow > m_actualRows) actualEndRow = m_actualRows; if (m_actualRows > actualEndRow) m_actualRows = actualEndRow; // reset to start of file if (!device.isSequential()) device.seek(0); DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << startRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header incl. start rows): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0 && !device.isSequential()) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode, lines); } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice & device, AbstractDataSource * dataSource, qint64 from, AbstractFileFilter::ImportMode importMode) { Q_ASSERT(dataSource != nullptr); FileDataSource* spreadsheet = dynamic_cast(dataSource); if (!m_prepared) { DEBUG("device is sequential = " << device.isSequential()); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) DEBUG("Device error = " << deviceError); if (deviceError) return 0; ////////// /////////////////////////// prepare import for spreadsheet spreadsheet->setUndoAware(false); //make the available columns undo unaware before we resize and rename them below, //the same will be done for new columns in this->resize(). for (int i = 0; i < spreadsheet->childCount(); i++) spreadsheet->child(i)->setUndoAware(false); qDebug() << "fds resizing!"; spreadsheet->removeColumns(0, 2); spreadsheet->clear(); spreadsheet->resize(importMode, vectorNames, m_actualCols); qDebug() << "fds resized to col: " << m_actualCols; qDebug() << "fds rowCount: " << spreadsheet->rowCount(); m_actualRows = 1; //also here we need a cheaper version of this if (!spreadsheet->keepLastValues()) spreadsheet->setRowCount(m_actualRows); else { spreadsheet->setRowCount(spreadsheet->keepNvalues()); m_actualRows = spreadsheet->keepNvalues(); } if (device.isSequential()) m_actualRows = 1; qDebug() << "fds rows resized to: " << m_actualRows; m_dataContainer.resize(m_actualCols); for (int n = 0; n < m_actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } - if (!device.isSequential()) - device.seek(device.size()); + if (!device.isSequential()) { + device.seek(device.size()); + spreadsheet->setBytesRead(device.size()); + } + m_prepared = true; qDebug() << "prepared!"; } qint64 bytesread = 0; // if there's data do be read if (device.bytesAvailable() > 0) { qDebug() << "got new data"; //move to the last read position, from == total bytes read //since the other source types are sequencial we cannot seek on them if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) device.seek(from); //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLinesForSampleRateNotTillEnd = 0; int newLinesTillEnd = 0; QVector newData; if (spreadsheet->readingType() != FileDataSource::ReadingType::TillEnd) { newData.reserve(spreadsheet->sampleRate()); newData.resize(spreadsheet->sampleRate()); } int newDataIdx = 0; while (!device.atEnd()) { if (spreadsheet->readingType() != FileDataSource::ReadingType::TillEnd) newData[newDataIdx++] = device.readLine(); else newData.push_back(device.readLine()); newLinesTillEnd++; if (spreadsheet->readingType() != FileDataSource::ReadingType::TillEnd) { newLinesForSampleRateNotTillEnd++; //for Continous reading and FromEnd we read sample rate number of lines if possible if (newLinesForSampleRateNotTillEnd == spreadsheet->sampleRate()) break; } } //we had less new lines than the sample rate specified if (spreadsheet->readingType() != FileDataSource::ReadingType::TillEnd) { qDebug() << "Removed empty lines: " << newData.removeAll(""); } //increase row count if we don't have a fixed size if (!spreadsheet->keepLastValues()) { if (spreadsheet->readingType() != FileDataSource::ReadingType::TillEnd) m_actualRows += qMin(newData.size(), spreadsheet->sampleRate()); else m_actualRows += newData.size(); } //back to the last read position before counting when reading from files if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) device.seek(from); const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); int currentRow; // indexes the position in the vector(column) //new rows/resize columns if we don't have a fixed size //TODO if the user changes this value..m_resizedToFixedSize..setResizedToFixedSize if (!spreadsheet->keepLastValues()) { if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); currentRow = spreadsheetRowCountBeforeResize; // indexes the position in the vector(column) // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } else { //when we have a fixed size we have to pop sampleRate number of lines if specified //here popping, setting currentRow currentRow = m_actualRows - 1 - qMin(spreadsheet->sampleRate(), newLinesTillEnd); for (int row = 0; row < qMin(spreadsheet->sampleRate(), newLinesTillEnd); ++row) { for (int col = 0; col < m_actualCols; ++col) { switch (columnModes[col]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(col)->data()); vector->pop_front(); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[col] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } } } // from the last row we read the new data in the spreadsheet qDebug() << "reading from line: " << currentRow; qDebug() <<"available bytes: " << device.bytesAvailable(); int linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; if (spreadsheet->keepLastValues()) linesToRead = qMin(spreadsheet->sampleRate(), newLinesTillEnd); qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows; if (spreadsheet->readingType() == FileDataSource::ReadingType::FromEnd) { if (newData.size() > spreadsheet->sampleRate()) newDataIdx = newData.size() - spreadsheet->sampleRate() - 1; else newDataIdx = 0; } for (int i = 0; i < linesToRead; ++i) { QString line; if (spreadsheet->readingType() == FileDataSource::ReadingType::FromEnd) line = newData.at(newDataIdx++); else line = newData.at(i); if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) bytesread += line.size(); qDebug() << "line bytes: " << line.size() << " line: " << line; qDebug() << "reading in row: " << currentRow; if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); for (int n = 0; n < m_actualCols; ++n) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : NAN); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = NAN; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = "NAN"; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } ////////// // set the comments for each of the columns if datasource is a spreadsheet const int rows = spreadsheet->rowCount(); for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); QString comment; switch (column->columnMode()) { case AbstractColumn::Numeric: comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); break; case AbstractColumn::Text: comment = i18np("text data, %1 element", "text data, %1 elements", rows); break; } column->setComment(comment); column->setSuppressDataChangedSignal(false); column->setChanged(); } } else qDebug() << "No new data available"; - ////////////////// + ////////////////// return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); Q_ASSERT(dataSource != nullptr); if (!m_prepared) { DEBUG("device is sequential = " << device.isSequential()); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) DEBUG("Device error = " << deviceError); if (deviceError == 1 && importMode == AbstractFileFilter::Replace && dataSource) dataSource->clear(); if (deviceError) return; // matrix data has only one column mode (which is not text) if (dynamic_cast(dataSource)) { auto mode = columnModes[0]; (mode == AbstractColumn::Text) ? mode = AbstractColumn::Numeric : 0; for (auto& c: columnModes) (c != mode) ? c = mode : 0; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows - startRow + 1, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); i++) { QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend the index if required //TODO: come up maybe with a solution with adding the index inside of the loop below, //without conversion to string, prepending to the list and then conversion back to integer. if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : NAN); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = NAN; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = "NAN"; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, dateTimeFormat, importMode); } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); i++) { QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend index if required if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); QStringList lineString; for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : NAN); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: // never happens case AbstractColumn::Day: break; } } else // missing columns in this line lineString += QLatin1String("NAN"); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: save data to ascii file } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement( "asciiFilter"); writer->writeAttribute( "commentCharacter", d->commentCharacter); writer->writeAttribute( "separatingCharacter", d->separatingCharacter); writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); writer->writeAttribute( "startRow", QString::number(d->startRow)); writer->writeAttribute( "endRow", QString::number(d->endRow)); writer->writeAttribute( "startColumn", QString::number(d->startColumn)); writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "asciiFilter") { reader->raiseError(i18n("no ascii filter element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("commentCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'commentCharacter'")); else d->commentCharacter = str; str = attribs.value("separatingCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'separatingCharacter'")); else d->separatingCharacter = str; str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'autoMode'")); else d->autoModeEnabled = str.toInt(); str = attribs.value("header").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'header'")); else d->headerEnabled = str.toInt(); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty str = attribs.value("simplifyWhitespaces").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'simplifyWhitespaces'")); else d->simplifyWhitespacesEnabled = str.toInt(); str = attribs.value("skipEmptyParts").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipEmptyParts'")); else d->skipEmptyParts = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startColumn'")); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endColumn'")); else d->endColumn = str.toInt(); return true; }