diff --git a/src/backend/datasources/FileDataSource.cpp b/src/backend/datasources/FileDataSource.cpp index 8d5855ba0..b2acc6291 100644 --- a/src/backend/datasources/FileDataSource.cpp +++ b/src/backend/datasources/FileDataSource.cpp @@ -1,809 +1,907 @@ /*************************************************************************** File : FileDataSource.cpp Project : LabPlot Description : Represents file data source -------------------------------------------------------------------- Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/FileDataSource.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/FITSFilter.h" #include "backend/datasources/filters/BinaryFilter.h" #include "backend/core/Project.h" #include "commonfrontend/spreadsheet/SpreadsheetView.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*! \class FileDataSource \brief Represents data stored in a file. Reading and writing is done with the help of appropriate I/O-filters. \ingroup datasources */ FileDataSource::FileDataSource(AbstractScriptingEngine* engine, const QString& name, bool loading) : Spreadsheet(engine, name, loading), m_fileType(Ascii), m_fileWatched(false), m_fileLinked(false), m_filter(0), m_fileSystemWatcher(0), m_updateTimer(new QTimer(this)), m_paused(false), m_prepared(false), m_newDataAvailable(false), m_bytesRead(0) { initActions(); connect(m_updateTimer, SIGNAL(timeout()), this, SLOT(read())); } FileDataSource::~FileDataSource() { if (m_filter) delete m_filter; if (m_fileSystemWatcher) delete m_fileSystemWatcher; if (m_file) delete m_file; if (m_localSocket) delete m_localSocket; if (m_tcpSocket) delete m_tcpSocket; if (m_serialPort) delete m_serialPort; delete m_updateTimer; } void FileDataSource::ready() { if (m_updateType == TimeInterval) m_updateTimer->start(m_updateFrequency); } void FileDataSource::initActions() { m_reloadAction = new QAction(QIcon::fromTheme("view-refresh"), i18n("Reload"), this); connect(m_reloadAction, SIGNAL(triggered()), this, SLOT(read())); m_toggleWatchAction = new QAction(i18n("Watch the file"), this); m_toggleWatchAction->setCheckable(true); connect(m_toggleWatchAction, SIGNAL(triggered()), this, SLOT(watchToggled())); m_toggleLinkAction = new QAction(i18n("Link the file"), this); m_toggleLinkAction->setCheckable(true); connect(m_toggleLinkAction, SIGNAL(triggered()), this, SLOT(linkToggled())); } //TODO make the view customizable (show as a spreadsheet or as a pure text file in an editor) QWidget *FileDataSource::view() const { if (!m_view) m_view = new SpreadsheetView(const_cast(this)); return m_view; } /*! * \brief Returns a list with the names of the available ports */ QStringList FileDataSource::availablePorts() { QStringList ports; qDebug() << "available ports count:" << QSerialPortInfo::availablePorts().size(); for(const QSerialPortInfo& sp : QSerialPortInfo::availablePorts()) { ports.append(sp.portName()); qDebug() << sp.description(); qDebug() << sp.manufacturer(); qDebug() << sp.portName(); qDebug() << sp.serialNumber(); qDebug() << sp.systemLocation(); } return ports; } /*! * \brief Returns a list with the supported baud rates */ QStringList FileDataSource::supportedBaudRates() { QStringList baudRates; for(const auto& baud : QSerialPortInfo::standardBaudRates()) baudRates.append(QString::number(baud)); return baudRates; } /*! * \brief Updates this data source at this moment */ void FileDataSource::updateNow() { m_updateTimer->stop(); read(); //restart the timer after update if (m_updateType == TimeInterval) m_updateTimer->start(m_updateFrequency); } /*! * \brief FileDataSource::stopReading */ //TODO: do we want this? void FileDataSource::stopReading() { if (m_updateType == TimeInterval) m_updateTimer->stop(); else if (m_updateType == NewData) disconnect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fileChanged())); } /*! * \brief Continue reading from the live data source after it was paused. */ void FileDataSource::continueReading() { m_paused = false; if (m_updateType == TimeInterval) m_updateTimer->start(m_updateFrequency); else if (m_updateType == NewData) connect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fileChanged())); } /*! * \brief Pause the reading of the live data source. */ void FileDataSource::pauseReading() { m_paused = true; if (m_updateType == TimeInterval) m_updateTimer->stop(); else if (m_updateType == NewData) disconnect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fileChanged())); } /*! returns the list with all supported data file formats. */ QStringList FileDataSource::fileTypes() { // see FileDataSource::FileType return (QStringList()<< i18n("ASCII data") << i18n("Binary data") << i18n("Image") << i18n("Hierarchical Data Format (HDF)") << i18n("Network Common Data Format (NetCDF)") // << "CDF" << i18n("Flexible Image Transport System Data Format (FITS)") // << i18n("Sound") ); } void FileDataSource::setFileName(const QString& name) { m_fileName=name; } QString FileDataSource::fileName() const { return m_fileName; } void FileDataSource::setFileType(const FileType type) { m_fileType = type; } FileDataSource::FileType FileDataSource::fileType() const { return m_fileType; } void FileDataSource::setFilter(AbstractFileFilter* f) { m_filter = f; } AbstractFileFilter* FileDataSource::filter() const { return m_filter; } /*! sets whether the file should be watched or not. In the first case the data source will be automatically updated on file changes. */ void FileDataSource::setFileWatched(const bool b) { m_fileWatched = b; } bool FileDataSource::isFileWatched() const { return m_fileWatched; } /*! * \brief Sets the serial port's baud rate * \param baudrate */ void FileDataSource::setBaudRate(const int baudrate) { m_baudRate = baudrate; } int FileDataSource::baudRate() const { return m_baudRate; } /*! * \brief Sets the source's update frequency to frequency * \param frequency */ void FileDataSource::setUpdateFrequency(const int frequency) { m_updateFrequency = frequency; m_updateTimer->start(m_updateFrequency); } int FileDataSource::updateFrequency() const { return m_updateFrequency; } /*! * \brief Sets how many values we should store * \param keepnvalues */ void FileDataSource::setKeepNvalues(const int keepnvalues) { m_keepNvalues = keepnvalues; } int FileDataSource::keepNvalues() const { return m_keepNvalues; } /*! * \brief Sets the network socket's port to port * \param port */ void FileDataSource::setPort(const int port) { m_port = port; } int FileDataSource::port() const { return m_port; } /*! * \brief Sets the serial port's name to name * \param name */ void FileDataSource::setSerialPort(const QString &name) { m_serialPortName = name; } QString FileDataSource::serialPortName() const { return m_serialPortName; } /*! * \brief Sets the sample rate to samplerate * \param samplerate */ void FileDataSource::setSampleRate(const int samplerate) { m_sampleRate = samplerate; } int FileDataSource::sampleRate() const { return m_sampleRate; } /*! * \brief Sets the source's type to sourcetype * \param sourcetype */ void FileDataSource::setSourceType(const SourceType sourcetype) { m_sourceType = sourcetype; } FileDataSource::SourceType FileDataSource::sourceType() const { return m_sourceType; } /*! * \brief Sets the source's reading type to readingType * \param readingType */ void FileDataSource::setReadingType(const ReadingType readingType) { m_readingType = readingType; } FileDataSource::ReadingType FileDataSource::readingType() const { return m_readingType; } /*! * \brief Sets the source's update type to updatetype * \param updatetype */ void FileDataSource::setUpdateType(const UpdateType updatetype) { if (updatetype == NewData) m_updateTimer->stop(); m_updateType = updatetype; } FileDataSource::UpdateType FileDataSource::updateType() const { return m_updateType; } /*! * \brief Sets the network socket's host * \param host */ void FileDataSource::setHost(const QString & host) { m_host = host; } QString FileDataSource::host() const { return m_host; } /*! sets whether only a link to the file is saved in the project file (\c b=true) or the whole content of the file (\c b=false). */ void FileDataSource::setFileLinked(const bool b) { m_fileLinked = b; } /*! returns \c true if only a link to the file is saved in the project file. \c false otherwise. */ bool FileDataSource::isFileLinked() const { return m_fileLinked; } QIcon FileDataSource::icon() const { QIcon icon; if (m_fileType == FileDataSource::Ascii) icon = QIcon::fromTheme("text-plain"); else if (m_fileType == FileDataSource::Binary) icon = QIcon::fromTheme("application-octet-stream"); else if (m_fileType == FileDataSource::Image) icon = QIcon::fromTheme("image-x-generic"); // TODO: HDF, NetCDF, FITS, etc. return icon; } QMenu* FileDataSource::createContextMenu() { QMenu* menu = AbstractPart::createContextMenu(); QAction* firstAction = 0; // if we're populating the context menu for the project explorer, then //there're already actions available there. Skip the first title-action //and insert the action at the beginning of the menu. if (menu->actions().size()>1) firstAction = menu->actions().at(1); if (!m_fileWatched) menu->insertAction(firstAction, m_reloadAction); m_toggleWatchAction->setChecked(m_fileWatched); menu->insertAction(firstAction, m_toggleWatchAction); m_toggleLinkAction->setChecked(m_fileLinked); menu->insertAction(firstAction, m_toggleLinkAction); return menu; } //############################################################################## //################################# SLOTS #################################### //############################################################################## void FileDataSource::read() { if (m_filter == nullptr) return; if (!m_prepared) { switch (m_sourceType) { case FileOrPipe: m_file = new QFile(m_fileName); break; case NetworkSocket: m_tcpSocket = new QTcpSocket; break; case LocalSocket: m_localSocket = new QLocalSocket; m_localSocket->setServerName(m_fileName); connect(m_localSocket, SIGNAL(readyRead()), this, SLOT(readyRead())); m_localSocket->connectToServer(QLocalSocket::ReadOnly); connect(m_localSocket, SIGNAL(error(QLocalSocket::LocalSocketError)), this, SLOT(localSocketError(QLocalSocket::LocalSocketError))); break; case SerialPort: m_serialPort = new QSerialPort; m_serialPort->setBaudRate(m_baudRate); m_serialPort->setPortName(m_serialPortName); connect(m_serialPort, SIGNAL(error(QSerialPort::SerialPortError)), this, SLOT(serialPortError(QSerialPort::SerialPortError))); connect(m_serialPort, SIGNAL(readyRead()), this, SLOT(readyRead())); break; } m_prepared = true; } qint64 bytes; switch (m_sourceType) { case FileOrPipe: switch (m_fileType) { case Ascii: qDebug() << "reading live ascii file.." ; bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_file, this, m_bytesRead, AbstractFileFilter::Replace); m_bytesRead += bytes; qDebug() << "read " << bytes << " bytes, in total: " << m_bytesRead; break; case Binary: //bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_file, this, m_bytesRead); m_bytesRead += bytes; default: break; } break; case NetworkSocket: break; case LocalSocket: if (m_newDataAvailable) { switch (m_fileType) { case Ascii: - bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_localSocket, this, m_bytesRead); - m_bytesRead += bytes; + dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_localSocket, this); break; case Binary: - // bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_localSocket, this, m_bytesRead); - m_bytesRead += bytes; + // dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_localSocket, this); break; default: break; } m_localSocket->abort(); m_localSocket->connectToServer(m_fileName, QLocalSocket::ReadOnly); m_newDataAvailable = false; } break; case SerialPort: if (m_newDataAvailable) { // copy data from buffer spreadsheet switch (m_fileType) { case Ascii: - bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_serialPort, this, m_bytesRead); - m_bytesRead += bytes; + dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_serialPort, this); break; case Binary: - // bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_serialPort, this, m_bytesRead); - m_bytesRead += bytes; + // dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_serialPort, this); break; default: break; } m_newDataAvailable = false; } break; } watch(); } //for sockets, serial port, network.. void FileDataSource::readyRead() { if (!m_newDataAvailable) m_newDataAvailable = true; //just like for files: the file system watcher emits the signal and we read on new data //here new data comes when this is called actually if (m_updateType == NewData) { read(); } } void FileDataSource::localSocketError(QLocalSocket::LocalSocketError socketError) { switch (socketError) { case QLocalSocket::ServerNotFoundError: QMessageBox::information(0, i18n("Local Socket Error"), i18n("The socket was not found. Please check the socket name.")); break; case QLocalSocket::ConnectionRefusedError: QMessageBox::information(0, i18n("LabPlot2"), i18n("The connection was refused by the peer")); break; case QLocalSocket::PeerClosedError: break; default: QMessageBox::information(0, i18n("LabPlot2"), i18n("The following error occurred: %1.") .arg(m_localSocket->errorString())); } } void FileDataSource::serialPortError(QSerialPort::SerialPortError serialPortError) { switch (serialPortError) { case QSerialPort::DeviceNotFoundError: break; case QSerialPort::PermissionError: break; case QSerialPort::OpenError: break; case QSerialPort::NotOpenError: break; case QSerialPort::ReadError: break; case QSerialPort::ResourceError: break; case QSerialPort::TimeoutError: break; default: break; } } void FileDataSource::fileChanged() { this->read(); } void FileDataSource::watchToggled() { m_fileWatched = !m_fileWatched; watch(); project()->setChanged(true); } void FileDataSource::linkToggled() { m_fileLinked = !m_fileLinked; project()->setChanged(true); } //watch the file upon reading for changes if required void FileDataSource::watch() { if (m_updateType == UpdateType::NewData) { if (m_fileWatched) { if (!m_fileSystemWatcher) { m_fileSystemWatcher = new QFileSystemWatcher; connect (m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fileChanged())); } if ( !m_fileSystemWatcher->files().contains(m_fileName) ) m_fileSystemWatcher->addPath(m_fileName); } else { if (m_fileSystemWatcher) m_fileSystemWatcher->removePath(m_fileName); } } } /*! returns a string containing the general information about the file \c name and some content specific information (number of columns and lines for ASCII, color-depth for images etc.). */ QString FileDataSource::fileInfoString(const QString &name) { QString infoString; QFileInfo fileInfo; QString fileTypeString; QIODevice *file = new QFile(name); QString fileName; if (name.at(0) != QDir::separator()) fileName = QDir::homePath() + QDir::separator() + name; else fileName = name; if(file==0) file = new QFile(fileName); if (file->open(QIODevice::ReadOnly)) { QStringList infoStrings; //general information about the file infoStrings << "" + fileName + "
"; fileInfo.setFile(fileName); infoStrings << i18n("Readable: %1", fileInfo.isReadable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Writable: %1", fileInfo.isWritable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Executable: %1", fileInfo.isExecutable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Created: %1", fileInfo.created().toString()); infoStrings << i18n("Last modified: %1", fileInfo.lastModified().toString()); infoStrings << i18n("Last read: %1", fileInfo.lastRead().toString()); infoStrings << i18n("Owner: %1", fileInfo.owner()); infoStrings << i18n("Group: %1", fileInfo.group()); infoStrings << i18n("Size: %1", i18np("%1 cByte", "%1 cBytes", fileInfo.size())); #ifdef HAVE_FITS if (fileName.endsWith(QLatin1String(".fits"))) { FITSFilter* fitsFilter = new FITSFilter; infoStrings << i18n("Images: %1", QString::number(fitsFilter->imagesCount(fileName) )); infoStrings << i18n("Tables: %1", QString::number(fitsFilter->tablesCount(fileName) )); delete fitsFilter; } #endif // file type and type specific information about the file #ifdef Q_OS_LINUX QProcess *proc = new QProcess(); QStringList args; args<<"-b"<start( "file", args); if(proc->waitForReadyRead(1000) == false) infoStrings << i18n("Could not open file %1 for reading.", fileName); else { fileTypeString = proc->readLine(); if( fileTypeString.contains(i18n("cannot open")) ) fileTypeString=""; else { fileTypeString.remove(fileTypeString.length()-1,1); // remove '\n' } } infoStrings << i18n("File type: %1", fileTypeString); #endif //TODO depending on the file type, generate additional information about the file: //Number of lines for ASCII, color-depth for images etc. Use the specific filters here. // port the old labplot1.6 code. if( fileTypeString.contains("ASCII")) { infoStrings << "
"; //TODO: consider choosen separator infoStrings << i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName)); infoStrings << i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); } infoString += infoStrings.join("
"); } else infoString += i18n("Could not open file %1 for reading.", fileName); return infoString; } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void FileDataSource::save(QXmlStreamWriter* writer) const { writer->writeStartElement("fileDataSource"); writeBasicAttributes(writer); writeCommentElement(writer); //general - writer->writeStartElement( "general" ); - writer->writeAttribute( "fileName", m_fileName ); - writer->writeAttribute( "fileType", QString::number(m_fileType) ); - writer->writeAttribute( "fileWatched", QString::number(m_fileWatched) ); - writer->writeAttribute( "fileLinked", QString::number(m_fileLinked) ); - writer->writeEndElement(); + writer->writeStartElement("general"); + writer->writeAttribute("fileName", m_fileName); + writer->writeAttribute("fileType", QString::number(m_fileType)); + writer->writeAttribute("fileWatched", QString::number(m_fileWatched)); + writer->writeAttribute("fileLinked", QString::number(m_fileLinked)); + writer->writeAttribute("updateType", QString::number(m_updateType)); + writer->writeAttribute("readingType", QString::number(m_readingType)); + writer->writeAttribute("sourceType", QString::number(m_sourceType)); + writer->writeAttribute("keepValues", QString::number(m_keepNvalues)); + + if (m_updateType == TimeInterval) { + writer->writeAttribute("updateFrequency", QString::number(m_updateFrequency)); + } + + if (m_readingType != TillEnd) { + writer->writeAttribute("sampleRate", QString::number(m_sampleRate)); + } + + switch (m_sourceType) { + case SerialPort: + writer->writeAttribute("baudRate", QString::number(m_baudRate)); + writer->writeAttribute("serialPortName", m_serialPortName); + + break; + case NetworkSocket: + writer->writeAttribute("host", m_host); + writer->writeAttribute("port", QString::number(m_port)); + break; + case FileOrPipe: + break; + case LocalSocket: + break; + default: + break; + } + + writer->writeEndElement(); //filter m_filter->save(writer); //columns if (!m_fileLinked) { foreach (Column * col, children(IncludeHidden)) col->save(writer); } writer->writeEndElement(); // "fileDataSource" } /*! Loads from XML. */ bool FileDataSource::load(XmlStreamReader* reader) { if(!reader->isStartElement() || reader->name() != "fileDataSource") { reader->raiseError(i18n("no fileDataSource element found")); return false; } if (!readBasicAttributes(reader)) return false; QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs; QString str; while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement() && reader->name() == "fileDataSource") break; if (!reader->isStartElement()) continue; if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if (reader->name() == "general") { attribs = reader->attributes(); str = attribs.value("fileName").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileName'")); else m_fileName = str; str = attribs.value("fileType").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileType'")); else m_fileType = (FileType)str.toInt(); str = attribs.value("fileWatched").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileWatched'")); else m_fileWatched = str.toInt(); str = attribs.value("fileLinked").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileLinked'")); else m_fileLinked = str.toInt(); + + str = attribs.value("updateType").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'updateType'")); + else + m_updateType = static_cast(str.toInt()); + + str = attribs.value("sourceType").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'sourceType'")); + else + m_sourceType = static_cast(str.toInt()); + + str = attribs.value("readingType").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'readingType'")); + else + m_readingType = static_cast(str.toInt()); + + if (m_updateType == TimeInterval) { + str = attribs.value("updateFrequency").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'updateFrequency'")); + else + m_updateFrequency = str.toInt(); + } + + if (m_readingType != TillEnd) { + str = attribs.value("sampleRate").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'sampleRate'")); + else + m_sampleRate = str.toInt(); + } + + switch (m_sourceType) { + case SerialPort: + str = attribs.value("baudRate").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'baudRate'")); + else + m_baudRate = str.toInt(); + + str = attribs.value("serialPortName").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'serialPortName'")); + else + m_serialPortName = str; + + break; + case NetworkSocket: + str = attribs.value("host").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'host'")); + else + m_host = str; + + str = attribs.value("port").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'port'")); + else + m_host = str; + break; + case FileOrPipe: + break; + case LocalSocket: + break; + default: + break; + } + } else if (reader->name() == "asciiFilter") { m_filter = new AsciiFilter(); if (!m_filter->load(reader)) return false; } else if(reader->name() == "column") { Column* column = new Column("", AbstractColumn::Text); if (!column->load(reader)) { delete column; setColumnCount(0); return false; } addChild(column); } else {// unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } //read the content of the file if it was only linked if (m_fileLinked) this->read(); return !reader->hasError(); } diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index f3aba0915..43c10ef2d 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,1048 +1,1042 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2017 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/FileDataSource.h" #include "backend/core/column/Column.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/lib/macros.h" #include #include #include #include #include #include /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() {} /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } +void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource * dataSource, AbstractFileFilter::ImportMode) { + d->readFromLiveDevice(device, dataSource); +} + qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from, AbstractFileFilter::ImportMode importMode, int lines) { return d->readFromLiveDevice(device, dataSource, from, importMode, lines); } /*! reads the content of the file \c fileName. */ QVector AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromFile(fileName, dataSource, importMode, lines); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; i++) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of lines"); return 0; } size_t lineCount = 0; while (!device.atEnd()) { device.readLine(); lineCount++; } //TODO: wc is much faster but not portable /* QElapsedTimer myTimer; myTimer.start(); QProcess wc; wc.start(QString("wc"), QStringList() << "-l" << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) lineCount = wc.readLine().split(' ')[0].toInt(); lineCount++; // last line not counted DEBUG(" Elapsed time counting lines : " << myTimer.elapsed() << " ms"); */ return lineCount; } /*! returns the number of lines in the device \c device or 0 if not available. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice &device) { // device.hasReadLine() always returns 0 for KFilterDev! if (device.isSequential()) return 0; size_t lineCount = 0; device.seek(0); while (!device.atEnd()) { device.readLine(); lineCount++; } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setVectorNames(const QString s) { d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner), commentCharacter("#"), separatingCharacter("auto"), createIndexEnabled(false), autoModeEnabled(true), headerEnabled(true), skipEmptyParts(false), simplifyWhitespacesEnabled(true), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_prepared(false), m_columnOffset(0) { } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd()) // empty file return 1; // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines firstLine = device.readLine(); if (device.atEnd()) return 1; } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, QString::SkipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, QString::SkipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); DEBUG("headerEnabled = " << headerEnabled); if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); startRow++; } if (createIndexEnabled) vectorNames.prepend("index"); // set range to read if (endColumn == -1) endColumn = firstLineStringList.size(); // last column m_actualCols = endColumn - startColumn + 1; if (createIndexEnabled) ++m_actualCols; //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ // this also resets position to start of file m_actualRows = AsciiFilter::lineNumber(device); // Find first data line (ignoring comment lines) DEBUG("Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; i++) { QString line = device.readLine(); if (device.atEnd()) return 1; if (line.startsWith(commentCharacter)) // ignore commented lines i--; } // parse first data line to determine data type for each column firstLine = device.readLine(); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("first data line : \'" << firstLine.toStdString() << '\''); firstLineStringList = firstLine.split(m_separator, QString::SkipEmptyParts); QDEBUG("first data line, parsed : " << firstLineStringList); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Numeric; col = 1; } for (const auto& valueString: firstLineStringList) { // only parse columns available in first data line if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } QDEBUG("column modes = " << columnModes); int actualEndRow = endRow; DEBUG("endRow = " << endRow); if (endRow == -1 || endRow > m_actualRows) actualEndRow = m_actualRows; if (m_actualRows > actualEndRow) m_actualRows = actualEndRow; // reset to start of file device.seek(0); DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << startRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header incl. start rows): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode, lines); } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice & device, AbstractDataSource * dataSource, qint64 from, AbstractFileFilter::ImportMode importMode, int lines) { Q_ASSERT(dataSource != nullptr); FileDataSource* spreadsheet = dynamic_cast(dataSource); if (!m_prepared) { DEBUG("device is sequential = " << device.isSequential()); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) DEBUG("Device error = " << deviceError); if (deviceError) return 0; ////////// /////////////////////////// prepare import for spreadsheet - /* m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows - startRow + 1, - m_actualCols, vectorNames, columnModes); -*/ - /*int actualRows, int actualCols, QStringList colNameList, QVector columnMode) {*/ - //DEBUG("create() rows = " << actualRows << " cols = " << actualCols); - //int columnOffset = 0; spreadsheet->setUndoAware(false); //make the available columns undo unaware before we resize and rename them below, //the same will be done for new columns in this->resize(). for (int i = 0; i < spreadsheet->childCount(); i++) spreadsheet->child(i)->setUndoAware(false); qDebug() << "fds resizing!"; - // needed cheaper version, we don't need undo stack for these 3 spreadsheet->removeColumns(0, 2); if (importMode == AbstractFileFilter::Replace) spreadsheet->clear(); spreadsheet->resize(importMode, vectorNames, m_actualCols); - qDebug() << "fds resized to col: " << m_actualCols; - // resize the spreadsheet - //if (importMode == AbstractFileFilter::Replace) { - // clear(); - // setRowCount(actualRows); - //} else { - - //} - qDebug() << "fds rowCount: " << spreadsheet->rowCount(); //also here we need a cheaper version of this if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); qDebug() << "fds rows resized to: " << m_actualRows; m_dataContainer.resize(m_actualCols); for (int n = 0; n < m_actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) spreadsheet->child(n)->setColumnMode(columnModes[n]); switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } m_prepared = true; qDebug() << "prepared!"; } qint64 bytesread = 0; // if there's data do be read if (device.bytesAvailable() > 0) { //move to the last read position, from == total bytes read - device.seek(from); + if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) { + device.seek(from); + } //count the new lines, increase actualrows on each //now we read all the new lines, if we want to use sample rate //then here we can do it, if we have actually sample rate number of lines :-? int newLines = 0; + while (!device.atEnd()) { device.readLine(); m_actualRows++; if (spreadsheet->readingType() != FileDataSource::ReadingType::TillEnd) { newLines++; //for Continous reading and FromEnd we read sample rate number of lines if possible if (newLines == spreadsheet->sampleRate()) break; } } - //back to the last read position before counting - device.seek(from); + //back to the last read position before counting when reading from files + if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) { + device.seek(from); + } + const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); //new rows if (spreadsheet->rowCount() < m_actualRows) spreadsheet->setRowCount(m_actualRows); // if we have fixed size, we do this only once in preparation, here we can use // m_prepared and we need something to decide whether it has a fixed size or increasing for (int n = 0; n < m_actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) switch (columnModes[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(spreadsheet->child(n)->data()); vector->reserve(m_actualRows); vector->resize(m_actualRows); m_dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } // from the last row we read the new data in the spreadsheet int currentRow = spreadsheetRowCountBeforeResize; // indexes the position in the vector(column) - if (lines == -1) - lines = m_actualRows; qDebug() << "reading from line: " << currentRow; qDebug() <<"available bytes: " << device.bytesAvailable(); const int linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows; for (int i = 0; i < /*qMin(lines, m_actualRows)*/linesToRead; ++i) { QString line = device.readLine(); - bytesread += line.size(); + if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) + bytesread += line.size(); qDebug() << "line bytes: " << line.size() << " line: " << line; qDebug() << "reading in row: " << currentRow; if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; QLocale locale(numberFormat); QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : NAN); qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = NAN; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = "NAN"; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; } ////////// // set the comments for each of the columns if datasource is a spreadsheet const int rows = spreadsheet->rowCount(); for (int n = 0; n < m_actualCols; ++n) { Column* column = spreadsheet->column(n); QString comment; switch (column->columnMode()) { case AbstractColumn::Numeric: comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); break; case AbstractColumn::Text: comment = i18np("text data, %1 element", "text data, %1 elements", rows); break; } column->setComment(comment); if (importMode == AbstractFileFilter::Replace) { column->setSuppressDataChangedSignal(false); column->setChanged(); } } } ////////////////// return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); Q_ASSERT(dataSource != nullptr); if (!m_prepared) { DEBUG("device is sequential = " << device.isSequential()); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) DEBUG("Device error = " << deviceError); if (deviceError == 1 && importMode == AbstractFileFilter::Replace && dataSource) dataSource->clear(); if (deviceError) return; // avoid text data in Matrix if (dynamic_cast(dataSource)) { for (auto& c: columnModes) if (c == AbstractColumn::Text) c = AbstractColumn::Numeric; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows - startRow + 1, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); i++) { QString line = device.readLine(); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend the index if required //TODO: come up maybe with a solution with adding the index inside of the loop below, //without conversion to string, prepending to the list and then conversion back to integer. if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : NAN); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); break; } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = NAN; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = "NAN"; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, dateTimeFormat, importMode); } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); i++) { QString line = device.readLine(); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend index if required if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); QStringList lineString; for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { bool isNumber; const double value = locale.toDouble(valueString, &isNumber); lineString += QString::number(isNumber ? value : NAN); break; } case AbstractColumn::DateTime: { const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); break; } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else // missing columns in this line lineString += QLatin1String("NAN"); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { - writer->writeStartElement( "asciiFilter" ); - writer->writeAttribute( "commentCharacter", d->commentCharacter ); - writer->writeAttribute( "separatingCharacter", d->separatingCharacter ); - writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled) ); - writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled) ); - writer->writeAttribute( "header", QString::number(d->headerEnabled) ); + writer->writeStartElement( "asciiFilter"); + writer->writeAttribute( "commentCharacter", d->commentCharacter); + writer->writeAttribute( "separatingCharacter", d->separatingCharacter); + writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); + writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); + writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); - writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts) ); - writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled) ); - writer->writeAttribute( "startRow", QString::number(d->startRow) ); - writer->writeAttribute( "endRow", QString::number(d->endRow) ); - writer->writeAttribute( "startColumn", QString::number(d->startColumn) ); - writer->writeAttribute( "endColumn", QString::number(d->endColumn) ); + writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); + writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); + writer->writeAttribute( "startRow", QString::number(d->startRow)); + writer->writeAttribute( "endRow", QString::number(d->endRow)); + writer->writeAttribute( "startColumn", QString::number(d->startColumn)); + writer->writeAttribute( "endColumn", QString::number(d->endColumn)); + writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "asciiFilter") { reader->raiseError(i18n("no ascii filter element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("commentCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'commentCharacter'")); else d->commentCharacter = str; str = attribs.value("separatingCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'separatingCharacter'")); else d->separatingCharacter = str; str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'autoMode'")); else d->autoModeEnabled = str.toInt(); str = attribs.value("header").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'header'")); else d->headerEnabled = str.toInt(); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty str = attribs.value("simplifyWhitespaces").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'simplifyWhitespaces'")); else d->simplifyWhitespacesEnabled = str.toInt(); str = attribs.value("skipEmptyParts").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipEmptyParts'")); else d->skipEmptyParts = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startColumn'")); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endColumn'")); else d->endColumn = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/AsciiFilter.h b/src/backend/datasources/filters/AsciiFilter.h index 1f112fa1c..b61180fb8 100644 --- a/src/backend/datasources/filters/AsciiFilter.h +++ b/src/backend/datasources/filters/AsciiFilter.h @@ -1,110 +1,111 @@ /*************************************************************************** File : AsciiFilter.h Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTER_H #define ASCIIFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" class QStringList; class QIODevice; class AsciiFilterPrivate; class AsciiFilter : public AbstractFileFilter { Q_OBJECT public: AsciiFilter(); ~AsciiFilter(); static QStringList separatorCharacters(); static QStringList commentCharacters(); static QStringList dataTypes(); static QStringList predefinedFilters(); static int columnNumber(const QString& fileName, const QString& separator = QString()); static size_t lineNumber(const QString& fileName); static size_t lineNumber(QIODevice&); // calculate number of lines if device supports it // read data from any device void readDataFromDevice(QIODevice& device, AbstractDataSource*, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); - + void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*dataSource, + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); qint64 readFromLiveDevice(QIODevice& device, AbstractDataSource*, - qint64 from, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + qint64 from = -1, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); // overloaded function to read from file QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); void loadFilterSettings(const QString&); void saveFilterSettings(const QString&) const; void setCommentCharacter(const QString&); QString commentCharacter() const; void setSeparatingCharacter(const QString&); QString separatingCharacter() const; void setDateTimeFormat(const QString&); QString dateTimeFormat() const; void setNumberFormat(QLocale::Language); QLocale::Language numberFormat() const; void setAutoModeEnabled(const bool); bool isAutoModeEnabled() const; void setHeaderEnabled(const bool); bool isHeaderEnabled() const; void setSkipEmptyParts(const bool); bool skipEmptyParts() const; void setSimplifyWhitespacesEnabled(const bool); bool simplifyWhitespacesEnabled() const; void setCreateIndexEnabled(const bool); void setVectorNames(const QString); QStringList vectorNames() const; QVector columnModes(); void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; virtual void save(QXmlStreamWriter*) const; virtual bool load(XmlStreamReader*); private: std::unique_ptr const d; friend class AsciiFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/AsciiFilterPrivate.h b/src/backend/datasources/filters/AsciiFilterPrivate.h index 418346b7f..f19d9fffe 100644 --- a/src/backend/datasources/filters/AsciiFilterPrivate.h +++ b/src/backend/datasources/filters/AsciiFilterPrivate.h @@ -1,84 +1,85 @@ /*************************************************************************** File : AsciiFilterPrivate.h Project : LabPlot Description : Private implementation class for AsciiFilter. -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTERPRIVATE_H #define ASCIIFILTERPRIVATE_H #include class KFilterDev; class AbstractDataSource; class AbstractColumn; class AsciiFilterPrivate { public: explicit AsciiFilterPrivate(AsciiFilter*); int prepareDeviceToRead(QIODevice&); void readDataFromDevice(QIODevice&, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); - - qint64 readFromLiveDevice(QIODevice&, AbstractDataSource*, qint64 from, + void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*, + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); + qint64 readFromLiveDevice(QIODevice&, AbstractDataSource*, qint64 from = -1, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); const AsciiFilter* q; QString commentCharacter; QString separatingCharacter; QString dateTimeFormat; QLocale::Language numberFormat; bool createIndexEnabled; bool autoModeEnabled; bool headerEnabled; bool skipEmptyParts; bool simplifyWhitespacesEnabled; QStringList vectorNames; QVector columnModes; int startRow; int endRow; int startColumn; int endColumn; private: QString m_separator; int m_actualRows; int m_actualCols; int m_prepared; int m_columnOffset; // indexes the "start column" in the datasource. Data will be imported starting from this column. QVector m_dataContainer; // pointers to the actual data containers void clearDataSource(AbstractDataSource*) const; }; #endif