diff --git a/src/backend/datasources/FileDataSource.cpp b/src/backend/datasources/FileDataSource.cpp index b2acc6291..5b9f72796 100644 --- a/src/backend/datasources/FileDataSource.cpp +++ b/src/backend/datasources/FileDataSource.cpp @@ -1,907 +1,904 @@ /*************************************************************************** File : FileDataSource.cpp Project : LabPlot Description : Represents file data source -------------------------------------------------------------------- Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/FileDataSource.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/FITSFilter.h" #include "backend/datasources/filters/BinaryFilter.h" #include "backend/core/Project.h" #include "commonfrontend/spreadsheet/SpreadsheetView.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*! \class FileDataSource \brief Represents data stored in a file. Reading and writing is done with the help of appropriate I/O-filters. \ingroup datasources */ FileDataSource::FileDataSource(AbstractScriptingEngine* engine, const QString& name, bool loading) : Spreadsheet(engine, name, loading), m_fileType(Ascii), m_fileWatched(false), m_fileLinked(false), m_filter(0), m_fileSystemWatcher(0), m_updateTimer(new QTimer(this)), m_paused(false), m_prepared(false), - m_newDataAvailable(false), - m_bytesRead(0) { + m_newDataAvailable(false), + m_bytesRead(0) { initActions(); connect(m_updateTimer, SIGNAL(timeout()), this, SLOT(read())); } FileDataSource::~FileDataSource() { if (m_filter) delete m_filter; if (m_fileSystemWatcher) delete m_fileSystemWatcher; if (m_file) delete m_file; if (m_localSocket) delete m_localSocket; if (m_tcpSocket) delete m_tcpSocket; if (m_serialPort) delete m_serialPort; delete m_updateTimer; } void FileDataSource::ready() { if (m_updateType == TimeInterval) m_updateTimer->start(m_updateFrequency); } void FileDataSource::initActions() { m_reloadAction = new QAction(QIcon::fromTheme("view-refresh"), i18n("Reload"), this); connect(m_reloadAction, SIGNAL(triggered()), this, SLOT(read())); m_toggleWatchAction = new QAction(i18n("Watch the file"), this); m_toggleWatchAction->setCheckable(true); connect(m_toggleWatchAction, SIGNAL(triggered()), this, SLOT(watchToggled())); m_toggleLinkAction = new QAction(i18n("Link the file"), this); m_toggleLinkAction->setCheckable(true); connect(m_toggleLinkAction, SIGNAL(triggered()), this, SLOT(linkToggled())); } //TODO make the view customizable (show as a spreadsheet or as a pure text file in an editor) QWidget *FileDataSource::view() const { if (!m_view) m_view = new SpreadsheetView(const_cast(this)); return m_view; } /*! * \brief Returns a list with the names of the available ports */ QStringList FileDataSource::availablePorts() { QStringList ports; qDebug() << "available ports count:" << QSerialPortInfo::availablePorts().size(); for(const QSerialPortInfo& sp : QSerialPortInfo::availablePorts()) { ports.append(sp.portName()); qDebug() << sp.description(); qDebug() << sp.manufacturer(); qDebug() << sp.portName(); qDebug() << sp.serialNumber(); qDebug() << sp.systemLocation(); } return ports; } /*! * \brief Returns a list with the supported baud rates */ QStringList FileDataSource::supportedBaudRates() { QStringList baudRates; for(const auto& baud : QSerialPortInfo::standardBaudRates()) baudRates.append(QString::number(baud)); return baudRates; } /*! * \brief Updates this data source at this moment */ void FileDataSource::updateNow() { m_updateTimer->stop(); read(); //restart the timer after update if (m_updateType == TimeInterval) m_updateTimer->start(m_updateFrequency); } /*! * \brief FileDataSource::stopReading */ //TODO: do we want this? void FileDataSource::stopReading() { if (m_updateType == TimeInterval) m_updateTimer->stop(); else if (m_updateType == NewData) disconnect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fileChanged())); } /*! * \brief Continue reading from the live data source after it was paused. */ void FileDataSource::continueReading() { m_paused = false; if (m_updateType == TimeInterval) m_updateTimer->start(m_updateFrequency); else if (m_updateType == NewData) connect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fileChanged())); } /*! * \brief Pause the reading of the live data source. */ void FileDataSource::pauseReading() { m_paused = true; if (m_updateType == TimeInterval) m_updateTimer->stop(); else if (m_updateType == NewData) disconnect(m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fileChanged())); } /*! returns the list with all supported data file formats. */ QStringList FileDataSource::fileTypes() { // see FileDataSource::FileType return (QStringList()<< i18n("ASCII data") << i18n("Binary data") << i18n("Image") << i18n("Hierarchical Data Format (HDF)") << i18n("Network Common Data Format (NetCDF)") // << "CDF" << i18n("Flexible Image Transport System Data Format (FITS)") // << i18n("Sound") ); } void FileDataSource::setFileName(const QString& name) { m_fileName=name; } QString FileDataSource::fileName() const { return m_fileName; } void FileDataSource::setFileType(const FileType type) { m_fileType = type; } FileDataSource::FileType FileDataSource::fileType() const { return m_fileType; } void FileDataSource::setFilter(AbstractFileFilter* f) { m_filter = f; } AbstractFileFilter* FileDataSource::filter() const { return m_filter; } /*! sets whether the file should be watched or not. In the first case the data source will be automatically updated on file changes. */ void FileDataSource::setFileWatched(const bool b) { m_fileWatched = b; } bool FileDataSource::isFileWatched() const { return m_fileWatched; } /*! * \brief Sets the serial port's baud rate * \param baudrate */ void FileDataSource::setBaudRate(const int baudrate) { m_baudRate = baudrate; } int FileDataSource::baudRate() const { return m_baudRate; } /*! * \brief Sets the source's update frequency to frequency * \param frequency */ void FileDataSource::setUpdateFrequency(const int frequency) { m_updateFrequency = frequency; - m_updateTimer->start(m_updateFrequency); + m_updateTimer->start(m_updateFrequency); } int FileDataSource::updateFrequency() const { return m_updateFrequency; } /*! * \brief Sets how many values we should store * \param keepnvalues */ void FileDataSource::setKeepNvalues(const int keepnvalues) { m_keepNvalues = keepnvalues; } int FileDataSource::keepNvalues() const { return m_keepNvalues; } /*! * \brief Sets the network socket's port to port * \param port */ void FileDataSource::setPort(const int port) { m_port = port; } int FileDataSource::port() const { return m_port; } /*! * \brief Sets the serial port's name to name * \param name */ void FileDataSource::setSerialPort(const QString &name) { m_serialPortName = name; } QString FileDataSource::serialPortName() const { return m_serialPortName; } /*! * \brief Sets the sample rate to samplerate * \param samplerate */ void FileDataSource::setSampleRate(const int samplerate) { m_sampleRate = samplerate; } int FileDataSource::sampleRate() const { return m_sampleRate; } /*! * \brief Sets the source's type to sourcetype * \param sourcetype */ void FileDataSource::setSourceType(const SourceType sourcetype) { m_sourceType = sourcetype; } FileDataSource::SourceType FileDataSource::sourceType() const { return m_sourceType; } /*! * \brief Sets the source's reading type to readingType * \param readingType */ void FileDataSource::setReadingType(const ReadingType readingType) { - m_readingType = readingType; + m_readingType = readingType; } FileDataSource::ReadingType FileDataSource::readingType() const { - return m_readingType; + return m_readingType; } /*! * \brief Sets the source's update type to updatetype * \param updatetype */ void FileDataSource::setUpdateType(const UpdateType updatetype) { if (updatetype == NewData) m_updateTimer->stop(); m_updateType = updatetype; } FileDataSource::UpdateType FileDataSource::updateType() const { return m_updateType; } /*! * \brief Sets the network socket's host * \param host */ void FileDataSource::setHost(const QString & host) { m_host = host; } QString FileDataSource::host() const { return m_host; } /*! sets whether only a link to the file is saved in the project file (\c b=true) or the whole content of the file (\c b=false). */ void FileDataSource::setFileLinked(const bool b) { m_fileLinked = b; } /*! returns \c true if only a link to the file is saved in the project file. \c false otherwise. */ bool FileDataSource::isFileLinked() const { return m_fileLinked; } QIcon FileDataSource::icon() const { QIcon icon; if (m_fileType == FileDataSource::Ascii) icon = QIcon::fromTheme("text-plain"); else if (m_fileType == FileDataSource::Binary) icon = QIcon::fromTheme("application-octet-stream"); else if (m_fileType == FileDataSource::Image) icon = QIcon::fromTheme("image-x-generic"); // TODO: HDF, NetCDF, FITS, etc. return icon; } QMenu* FileDataSource::createContextMenu() { QMenu* menu = AbstractPart::createContextMenu(); QAction* firstAction = 0; // if we're populating the context menu for the project explorer, then //there're already actions available there. Skip the first title-action //and insert the action at the beginning of the menu. if (menu->actions().size()>1) firstAction = menu->actions().at(1); if (!m_fileWatched) menu->insertAction(firstAction, m_reloadAction); m_toggleWatchAction->setChecked(m_fileWatched); menu->insertAction(firstAction, m_toggleWatchAction); m_toggleLinkAction->setChecked(m_fileLinked); menu->insertAction(firstAction, m_toggleLinkAction); return menu; } //############################################################################## //################################# SLOTS #################################### //############################################################################## void FileDataSource::read() { if (m_filter == nullptr) return; if (!m_prepared) { switch (m_sourceType) { case FileOrPipe: m_file = new QFile(m_fileName); break; case NetworkSocket: m_tcpSocket = new QTcpSocket; break; case LocalSocket: m_localSocket = new QLocalSocket; m_localSocket->setServerName(m_fileName); - connect(m_localSocket, SIGNAL(readyRead()), this, SLOT(readyRead())); - m_localSocket->connectToServer(QLocalSocket::ReadOnly); - connect(m_localSocket, SIGNAL(error(QLocalSocket::LocalSocketError)), this, SLOT(localSocketError(QLocalSocket::LocalSocketError))); + connect(m_localSocket, SIGNAL(readyRead()), this, SLOT(readyRead())); + m_localSocket->connectToServer(QLocalSocket::ReadOnly); + connect(m_localSocket, SIGNAL(error(QLocalSocket::LocalSocketError)), this, SLOT(localSocketError(QLocalSocket::LocalSocketError))); break; case SerialPort: m_serialPort = new QSerialPort; m_serialPort->setBaudRate(m_baudRate); m_serialPort->setPortName(m_serialPortName); connect(m_serialPort, SIGNAL(error(QSerialPort::SerialPortError)), this, SLOT(serialPortError(QSerialPort::SerialPortError))); - connect(m_serialPort, SIGNAL(readyRead()), this, SLOT(readyRead())); + connect(m_serialPort, SIGNAL(readyRead()), this, SLOT(readyRead())); break; } m_prepared = true; - } - qint64 bytes; - - switch (m_sourceType) { - case FileOrPipe: - switch (m_fileType) { - case Ascii: - qDebug() << "reading live ascii file.." ; - bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_file, this, m_bytesRead, AbstractFileFilter::Replace); - m_bytesRead += bytes; - qDebug() << "read " << bytes << " bytes, in total: " << m_bytesRead; - - break; - case Binary: - //bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_file, this, m_bytesRead); - m_bytesRead += bytes; - default: - break; - } - break; - case NetworkSocket: - break; - case LocalSocket: - if (m_newDataAvailable) { - switch (m_fileType) { - case Ascii: - dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_localSocket, this); - break; - case Binary: - // dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_localSocket, this); - break; - default: - break; - } - m_localSocket->abort(); - m_localSocket->connectToServer(m_fileName, QLocalSocket::ReadOnly); - m_newDataAvailable = false; - } - break; - case SerialPort: - if (m_newDataAvailable) { - // copy data from buffer spreadsheet - switch (m_fileType) { - case Ascii: - dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_serialPort, this); - break; - case Binary: - // dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_serialPort, this); - break; - - default: - break; - } - m_newDataAvailable = false; - } - break; - } + } + qint64 bytes; + + switch (m_sourceType) { + case FileOrPipe: + switch (m_fileType) { + case Ascii: + qDebug() << "reading live ascii file.." ; + bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_file, this, m_bytesRead, AbstractFileFilter::Replace); + m_bytesRead += bytes; + qDebug() << "read " << bytes << " bytes, in total: " << m_bytesRead; + + break; + case Binary: + //bytes = dynamic_cast(m_filter)->readFromLiveDevice(*m_file, this, m_bytesRead); + m_bytesRead += bytes; + default: + break; + } + break; + case NetworkSocket: + break; + case LocalSocket: + if (m_newDataAvailable) { + switch (m_fileType) { + case Ascii: + dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_localSocket, this); + break; + case Binary: + // dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_localSocket, this); + break; + default: + break; + } + m_localSocket->abort(); + m_localSocket->connectToServer(m_fileName, QLocalSocket::ReadOnly); + m_newDataAvailable = false; + } + break; + case SerialPort: + if (m_newDataAvailable) { + // copy data from buffer spreadsheet + switch (m_fileType) { + case Ascii: + dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_serialPort, this); + break; + case Binary: + // dynamic_cast(m_filter)->readFromLiveDeviceNotFile(*m_serialPort, this); + break; + + default: + break; + } + m_newDataAvailable = false; + } + break; + } watch(); } //for sockets, serial port, network.. void FileDataSource::readyRead() { if (!m_newDataAvailable) m_newDataAvailable = true; - //just like for files: the file system watcher emits the signal and we read on new data - //here new data comes when this is called actually - if (m_updateType == NewData) { - read(); - } + //just like for files: the file system watcher emits the signal and we read on new data + //here new data comes when this is called actually + if (m_updateType == NewData) + read(); } void FileDataSource::localSocketError(QLocalSocket::LocalSocketError socketError) { switch (socketError) { case QLocalSocket::ServerNotFoundError: QMessageBox::information(0, i18n("Local Socket Error"), i18n("The socket was not found. Please check the socket name.")); break; case QLocalSocket::ConnectionRefusedError: - QMessageBox::information(0, i18n("LabPlot2"), - i18n("The connection was refused by the peer")); + QMessageBox::information(0, i18n("LabPlot2"), + i18n("The connection was refused by the peer")); break; case QLocalSocket::PeerClosedError: break; default: - QMessageBox::information(0, i18n("LabPlot2"), - i18n("The following error occurred: %1.") + QMessageBox::information(0, i18n("LabPlot2"), + i18n("The following error occurred: %1.") .arg(m_localSocket->errorString())); } } void FileDataSource::serialPortError(QSerialPort::SerialPortError serialPortError) { switch (serialPortError) { case QSerialPort::DeviceNotFoundError: break; case QSerialPort::PermissionError: break; case QSerialPort::OpenError: break; case QSerialPort::NotOpenError: break; case QSerialPort::ReadError: break; case QSerialPort::ResourceError: break; case QSerialPort::TimeoutError: break; default: break; } } void FileDataSource::fileChanged() { this->read(); } void FileDataSource::watchToggled() { m_fileWatched = !m_fileWatched; watch(); project()->setChanged(true); } void FileDataSource::linkToggled() { m_fileLinked = !m_fileLinked; project()->setChanged(true); } //watch the file upon reading for changes if required void FileDataSource::watch() { if (m_updateType == UpdateType::NewData) { if (m_fileWatched) { if (!m_fileSystemWatcher) { m_fileSystemWatcher = new QFileSystemWatcher; connect (m_fileSystemWatcher, SIGNAL(fileChanged(QString)), this, SLOT(fileChanged())); } if ( !m_fileSystemWatcher->files().contains(m_fileName) ) m_fileSystemWatcher->addPath(m_fileName); } else { if (m_fileSystemWatcher) m_fileSystemWatcher->removePath(m_fileName); } } } /*! returns a string containing the general information about the file \c name and some content specific information (number of columns and lines for ASCII, color-depth for images etc.). */ QString FileDataSource::fileInfoString(const QString &name) { QString infoString; QFileInfo fileInfo; QString fileTypeString; QIODevice *file = new QFile(name); QString fileName; if (name.at(0) != QDir::separator()) fileName = QDir::homePath() + QDir::separator() + name; else fileName = name; if(file==0) file = new QFile(fileName); if (file->open(QIODevice::ReadOnly)) { QStringList infoStrings; //general information about the file infoStrings << "" + fileName + "
"; fileInfo.setFile(fileName); infoStrings << i18n("Readable: %1", fileInfo.isReadable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Writable: %1", fileInfo.isWritable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Executable: %1", fileInfo.isExecutable() ? i18n("yes") : i18n("no")); infoStrings << i18n("Created: %1", fileInfo.created().toString()); infoStrings << i18n("Last modified: %1", fileInfo.lastModified().toString()); infoStrings << i18n("Last read: %1", fileInfo.lastRead().toString()); infoStrings << i18n("Owner: %1", fileInfo.owner()); infoStrings << i18n("Group: %1", fileInfo.group()); infoStrings << i18n("Size: %1", i18np("%1 cByte", "%1 cBytes", fileInfo.size())); #ifdef HAVE_FITS if (fileName.endsWith(QLatin1String(".fits"))) { FITSFilter* fitsFilter = new FITSFilter; infoStrings << i18n("Images: %1", QString::number(fitsFilter->imagesCount(fileName) )); infoStrings << i18n("Tables: %1", QString::number(fitsFilter->tablesCount(fileName) )); delete fitsFilter; } #endif // file type and type specific information about the file #ifdef Q_OS_LINUX QProcess *proc = new QProcess(); QStringList args; args<<"-b"<start( "file", args); if(proc->waitForReadyRead(1000) == false) infoStrings << i18n("Could not open file %1 for reading.", fileName); else { fileTypeString = proc->readLine(); if( fileTypeString.contains(i18n("cannot open")) ) fileTypeString=""; else { fileTypeString.remove(fileTypeString.length()-1,1); // remove '\n' } } infoStrings << i18n("File type: %1", fileTypeString); #endif //TODO depending on the file type, generate additional information about the file: //Number of lines for ASCII, color-depth for images etc. Use the specific filters here. // port the old labplot1.6 code. if( fileTypeString.contains("ASCII")) { infoStrings << "
"; //TODO: consider choosen separator infoStrings << i18n("Number of columns: %1", AsciiFilter::columnNumber(fileName)); infoStrings << i18n("Number of lines: %1", AsciiFilter::lineNumber(fileName)); } infoString += infoStrings.join("
"); } else infoString += i18n("Could not open file %1 for reading.", fileName); return infoString; } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void FileDataSource::save(QXmlStreamWriter* writer) const { writer->writeStartElement("fileDataSource"); writeBasicAttributes(writer); writeCommentElement(writer); //general - writer->writeStartElement("general"); - writer->writeAttribute("fileName", m_fileName); - writer->writeAttribute("fileType", QString::number(m_fileType)); - writer->writeAttribute("fileWatched", QString::number(m_fileWatched)); - writer->writeAttribute("fileLinked", QString::number(m_fileLinked)); - writer->writeAttribute("updateType", QString::number(m_updateType)); - writer->writeAttribute("readingType", QString::number(m_readingType)); - writer->writeAttribute("sourceType", QString::number(m_sourceType)); - writer->writeAttribute("keepValues", QString::number(m_keepNvalues)); - - if (m_updateType == TimeInterval) { - writer->writeAttribute("updateFrequency", QString::number(m_updateFrequency)); - } - - if (m_readingType != TillEnd) { - writer->writeAttribute("sampleRate", QString::number(m_sampleRate)); - } - - switch (m_sourceType) { - case SerialPort: - writer->writeAttribute("baudRate", QString::number(m_baudRate)); - writer->writeAttribute("serialPortName", m_serialPortName); - - break; - case NetworkSocket: - writer->writeAttribute("host", m_host); - writer->writeAttribute("port", QString::number(m_port)); - break; - case FileOrPipe: - break; - case LocalSocket: - break; - default: - break; - } - - writer->writeEndElement(); + writer->writeStartElement("general"); + writer->writeAttribute("fileName", m_fileName); + writer->writeAttribute("fileType", QString::number(m_fileType)); + writer->writeAttribute("fileWatched", QString::number(m_fileWatched)); + writer->writeAttribute("fileLinked", QString::number(m_fileLinked)); + writer->writeAttribute("updateType", QString::number(m_updateType)); + writer->writeAttribute("readingType", QString::number(m_readingType)); + writer->writeAttribute("sourceType", QString::number(m_sourceType)); + writer->writeAttribute("keepValues", QString::number(m_keepNvalues)); + + if (m_updateType == TimeInterval) + writer->writeAttribute("updateFrequency", QString::number(m_updateFrequency)); + + if (m_readingType != TillEnd) + writer->writeAttribute("sampleRate", QString::number(m_sampleRate)); + + switch (m_sourceType) { + case SerialPort: + writer->writeAttribute("baudRate", QString::number(m_baudRate)); + writer->writeAttribute("serialPortName", m_serialPortName); + + break; + case NetworkSocket: + writer->writeAttribute("host", m_host); + writer->writeAttribute("port", QString::number(m_port)); + break; + case FileOrPipe: + break; + case LocalSocket: + break; + default: + break; + } + + writer->writeEndElement(); //filter m_filter->save(writer); //columns if (!m_fileLinked) { foreach (Column * col, children(IncludeHidden)) col->save(writer); } writer->writeEndElement(); // "fileDataSource" } /*! Loads from XML. */ bool FileDataSource::load(XmlStreamReader* reader) { if(!reader->isStartElement() || reader->name() != "fileDataSource") { reader->raiseError(i18n("no fileDataSource element found")); return false; } if (!readBasicAttributes(reader)) return false; QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs; QString str; while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement() && reader->name() == "fileDataSource") break; if (!reader->isStartElement()) continue; if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if (reader->name() == "general") { attribs = reader->attributes(); str = attribs.value("fileName").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileName'")); else m_fileName = str; str = attribs.value("fileType").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileType'")); else m_fileType = (FileType)str.toInt(); str = attribs.value("fileWatched").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileWatched'")); else m_fileWatched = str.toInt(); str = attribs.value("fileLinked").toString(); if(str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'fileLinked'")); else m_fileLinked = str.toInt(); - str = attribs.value("updateType").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'updateType'")); - else - m_updateType = static_cast(str.toInt()); - - str = attribs.value("sourceType").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'sourceType'")); - else - m_sourceType = static_cast(str.toInt()); - - str = attribs.value("readingType").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'readingType'")); - else - m_readingType = static_cast(str.toInt()); - - if (m_updateType == TimeInterval) { - str = attribs.value("updateFrequency").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'updateFrequency'")); - else - m_updateFrequency = str.toInt(); - } - - if (m_readingType != TillEnd) { - str = attribs.value("sampleRate").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'sampleRate'")); - else - m_sampleRate = str.toInt(); - } - - switch (m_sourceType) { - case SerialPort: - str = attribs.value("baudRate").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'baudRate'")); - else - m_baudRate = str.toInt(); - - str = attribs.value("serialPortName").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'serialPortName'")); - else - m_serialPortName = str; - - break; - case NetworkSocket: - str = attribs.value("host").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'host'")); - else - m_host = str; - - str = attribs.value("port").toString(); - if(str.isEmpty()) - reader->raiseWarning(attributeWarning.arg("'port'")); - else - m_host = str; - break; - case FileOrPipe: - break; - case LocalSocket: - break; - default: - break; - } + str = attribs.value("updateType").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'updateType'")); + else + m_updateType = static_cast(str.toInt()); + + str = attribs.value("sourceType").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'sourceType'")); + else + m_sourceType = static_cast(str.toInt()); + + str = attribs.value("readingType").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'readingType'")); + else + m_readingType = static_cast(str.toInt()); + + if (m_updateType == TimeInterval) { + str = attribs.value("updateFrequency").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'updateFrequency'")); + else + m_updateFrequency = str.toInt(); + } + + if (m_readingType != TillEnd) { + str = attribs.value("sampleRate").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'sampleRate'")); + else + m_sampleRate = str.toInt(); + } + + switch (m_sourceType) { + case SerialPort: + str = attribs.value("baudRate").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'baudRate'")); + else + m_baudRate = str.toInt(); + + str = attribs.value("serialPortName").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'serialPortName'")); + else + m_serialPortName = str; + + break; + case NetworkSocket: + str = attribs.value("host").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'host'")); + else + m_host = str; + + str = attribs.value("port").toString(); + if(str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'port'")); + else + m_host = str; + break; + case FileOrPipe: + break; + case LocalSocket: + break; + default: + break; + } } else if (reader->name() == "asciiFilter") { m_filter = new AsciiFilter(); if (!m_filter->load(reader)) return false; } else if(reader->name() == "column") { Column* column = new Column("", AbstractColumn::Text); if (!column->load(reader)) { delete column; setColumnCount(0); return false; } addChild(column); } else {// unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } //read the content of the file if it was only linked if (m_fileLinked) this->read(); return !reader->hasError(); } diff --git a/src/backend/datasources/filters/AsciiFilter.cpp b/src/backend/datasources/filters/AsciiFilter.cpp index 43c10ef2d..f6f0a3603 100644 --- a/src/backend/datasources/filters/AsciiFilter.cpp +++ b/src/backend/datasources/filters/AsciiFilter.cpp @@ -1,1042 +1,1040 @@ /*************************************************************************** File : AsciiFilter.cpp Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2017 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/FileDataSource.h" #include "backend/core/column/Column.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/AsciiFilterPrivate.h" #include "backend/lib/macros.h" #include #include #include #include #include #include /*! \class AsciiFilter \brief Manages the import/export of data organized as columns (vectors) from/to an ASCII-file. \ingroup datasources */ AsciiFilter::AsciiFilter() : AbstractFileFilter(), d(new AsciiFilterPrivate(this)) {} AsciiFilter::~AsciiFilter() {} /*! reads the content of the device \c device. */ void AsciiFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromDevice(device, dataSource, importMode, lines); } void AsciiFilter::readFromLiveDeviceNotFile(QIODevice &device, AbstractDataSource * dataSource, AbstractFileFilter::ImportMode) { - d->readFromLiveDevice(device, dataSource); + d->readFromLiveDevice(device, dataSource); } qint64 AsciiFilter::readFromLiveDevice(QIODevice& device, AbstractDataSource* dataSource, qint64 from, AbstractFileFilter::ImportMode importMode, int lines) { - return d->readFromLiveDevice(device, dataSource, from, importMode, lines); + return d->readFromLiveDevice(device, dataSource, from, importMode, lines); } /*! reads the content of the file \c fileName. */ QVector AsciiFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { d->readDataFromFile(fileName, dataSource, importMode, lines); return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } QVector AsciiFilter::preview(const QString& fileName, int lines) { return d->preview(fileName, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ //void AsciiFilter::read(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode) { // d->read(fileName, dataSource, importMode); //} /*! writes the content of the data source \c dataSource to the file \c fileName. */ void AsciiFilter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! loads the predefined filter settings for \c filterName */ void AsciiFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void AsciiFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /*! returns the list with the names of all saved (system wide or user defined) filter settings. */ QStringList AsciiFilter::predefinedFilters() { return QStringList(); } /*! returns the list of all predefined separator characters. */ QStringList AsciiFilter::separatorCharacters() { return (QStringList() << "auto" << "TAB" << "SPACE" << "," << ";" << ":" << ",TAB" << ";TAB" << ":TAB" << ",SPACE" << ";SPACE" << ":SPACE"); } /*! returns the list of all predefined comment characters. */ QStringList AsciiFilter::commentCharacters() { return (QStringList() << "#" << "!" << "//" << "+" << "c" << ":" << ";"); } /*! returns the list of all predefined data types. */ QStringList AsciiFilter::dataTypes() { const QMetaObject& mo = AbstractColumn::staticMetaObject; const QMetaEnum& me = mo.enumerator(mo.indexOfEnumerator("ColumnMode")); QStringList list; for (int i = 0; i <= 100; i++) // me.keyCount() does not work because we have holes in enum if (me.valueToKey(i)) list << me.valueToKey(i); return list; } /*! returns the number of columns in the file \c fileName. */ int AsciiFilter::columnNumber(const QString& fileName, const QString& separator) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of columns"); return -1; } QString line = device.readLine(); line.remove(QRegExp("[\\n\\r]")); QStringList lineStringList; if (separator.length() > 0) lineStringList = line.split(separator); else lineStringList = line.split(QRegExp("\\s+")); DEBUG("number of columns : " << lineStringList.size()); return lineStringList.size(); } size_t AsciiFilter::lineNumber(const QString& fileName) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) { DEBUG("Could not open file " << fileName.toStdString() << " for determining number of lines"); return 0; } size_t lineCount = 0; while (!device.atEnd()) { device.readLine(); lineCount++; } //TODO: wc is much faster but not portable /* QElapsedTimer myTimer; myTimer.start(); QProcess wc; wc.start(QString("wc"), QStringList() << "-l" << fileName); size_t lineCount = 0; while (wc.waitForReadyRead()) lineCount = wc.readLine().split(' ')[0].toInt(); lineCount++; // last line not counted DEBUG(" Elapsed time counting lines : " << myTimer.elapsed() << " ms"); */ return lineCount; } /*! returns the number of lines in the device \c device or 0 if not available. resets the position to 0! */ size_t AsciiFilter::lineNumber(QIODevice &device) { // device.hasReadLine() always returns 0 for KFilterDev! if (device.isSequential()) return 0; size_t lineCount = 0; device.seek(0); while (!device.atEnd()) { device.readLine(); lineCount++; } device.seek(0); return lineCount; } void AsciiFilter::setCommentCharacter(const QString& s) { d->commentCharacter = s; } QString AsciiFilter::commentCharacter() const { return d->commentCharacter; } void AsciiFilter::setSeparatingCharacter(const QString& s) { d->separatingCharacter = s; } QString AsciiFilter::separatingCharacter() const { return d->separatingCharacter; } void AsciiFilter::setDateTimeFormat(const QString &f) { d->dateTimeFormat = f; } QString AsciiFilter::dateTimeFormat() const { return d->dateTimeFormat; } void AsciiFilter::setNumberFormat(QLocale::Language lang) { d->numberFormat = lang; } QLocale::Language AsciiFilter::numberFormat() const { return d->numberFormat; } void AsciiFilter::setAutoModeEnabled(const bool b) { d->autoModeEnabled = b; } bool AsciiFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } void AsciiFilter::setHeaderEnabled(const bool b) { d->headerEnabled = b; } bool AsciiFilter::isHeaderEnabled() const { return d->headerEnabled; } void AsciiFilter::setSkipEmptyParts(const bool b) { d->skipEmptyParts = b; } bool AsciiFilter::skipEmptyParts() const { return d->skipEmptyParts; } void AsciiFilter::setCreateIndexEnabled(bool b) { d->createIndexEnabled = b; } void AsciiFilter::setSimplifyWhitespacesEnabled(bool b) { d->simplifyWhitespacesEnabled = b; } bool AsciiFilter::simplifyWhitespacesEnabled() const { return d->simplifyWhitespacesEnabled; } void AsciiFilter::setVectorNames(const QString s) { d->vectorNames = s.simplified().split(' '); } QStringList AsciiFilter::vectorNames() const { return d->vectorNames; } QVector AsciiFilter::columnModes() { return d->columnModes; } void AsciiFilter::setStartRow(const int r) { d->startRow = r; } int AsciiFilter::startRow() const { return d->startRow; } void AsciiFilter::setEndRow(const int r) { d->endRow = r; } int AsciiFilter::endRow() const { return d->endRow; } void AsciiFilter::setStartColumn(const int c) { d->startColumn = c; } int AsciiFilter::startColumn() const { return d->startColumn; } void AsciiFilter::setEndColumn(const int c) { d->endColumn = c; } int AsciiFilter::endColumn() const { return d->endColumn; } //##################################################################### //################### Private implementation ########################## //##################################################################### AsciiFilterPrivate::AsciiFilterPrivate(AsciiFilter* owner) : q(owner), commentCharacter("#"), separatingCharacter("auto"), - createIndexEnabled(false), autoModeEnabled(true), headerEnabled(true), skipEmptyParts(false), simplifyWhitespacesEnabled(true), + createIndexEnabled(false), startRow(1), endRow(-1), startColumn(1), endColumn(-1), m_prepared(false), m_columnOffset(0) { } /*! * returns -1 if the device couldn't be opened, 1 if the current read position in the device is at the end and 0 otherwise. */ int AsciiFilterPrivate::prepareDeviceToRead(QIODevice& device) { if (!device.open(QIODevice::ReadOnly)) return -1; if (device.atEnd()) // empty file return 1; // Parse the first line: // Determine the number of columns, create the columns and use (if selected) the first row to name them QString firstLine; do { // skip comment lines firstLine = device.readLine(); if (device.atEnd()) return 1; } while (firstLine.startsWith(commentCharacter)); DEBUG(" device position after first line and comments = " << device.pos()); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("First line: \'" << firstLine.toStdString() << '\''); // determine separator and split first line QStringList firstLineStringList; if (separatingCharacter == "auto") { DEBUG("automatic separator"); QRegExp regExp("(\\s+)|(,\\s+)|(;\\s+)|(:\\s+)"); firstLineStringList = firstLine.split(regExp, QString::SkipEmptyParts); if (!firstLineStringList.isEmpty()) { int length1 = firstLineStringList.at(0).length(); if (firstLineStringList.size() > 1) { int pos2 = firstLine.indexOf(firstLineStringList.at(1), length1); m_separator = firstLine.mid(length1, pos2 - length1); } else { //old: separator = line.right(line.length() - length1); m_separator = ' '; } } } else { // use given separator // replace symbolic "TAB" with '\t' m_separator = separatingCharacter.replace(QLatin1String("TAB"), "\t", Qt::CaseInsensitive); // replace symbolic "SPACE" with ' ' m_separator = m_separator.replace(QLatin1String("SPACE"), QLatin1String(" "), Qt::CaseInsensitive); firstLineStringList = firstLine.split(m_separator, QString::SkipEmptyParts); } DEBUG("separator: \'" << m_separator.toStdString() << '\''); DEBUG("number of columns: " << firstLineStringList.size()); DEBUG("headerEnabled = " << headerEnabled); if (headerEnabled) { // use first line to name vectors vectorNames = firstLineStringList; QDEBUG("vector names =" << vectorNames); startRow++; } if (createIndexEnabled) vectorNames.prepend("index"); // set range to read if (endColumn == -1) endColumn = firstLineStringList.size(); // last column m_actualCols = endColumn - startColumn + 1; if (createIndexEnabled) ++m_actualCols; //TEST: readline-seek-readline fails /* qint64 testpos = device.pos(); DEBUG("read data line @ pos " << testpos << " : " << device.readLine().toStdString()); device.seek(testpos); testpos = device.pos(); DEBUG("read data line again @ pos " << testpos << " : " << device.readLine().toStdString()); */ // this also resets position to start of file m_actualRows = AsciiFilter::lineNumber(device); // Find first data line (ignoring comment lines) DEBUG("Skipping " << startRow - 1 << " lines"); for (int i = 0; i < startRow - 1; i++) { QString line = device.readLine(); if (device.atEnd()) return 1; if (line.startsWith(commentCharacter)) // ignore commented lines i--; } // parse first data line to determine data type for each column firstLine = device.readLine(); firstLine.remove(QRegExp("[\\n\\r]")); // remove any newline if (simplifyWhitespacesEnabled) firstLine = firstLine.simplified(); DEBUG("first data line : \'" << firstLine.toStdString() << '\''); firstLineStringList = firstLine.split(m_separator, QString::SkipEmptyParts); QDEBUG("first data line, parsed : " << firstLineStringList); columnModes.resize(m_actualCols); int col = 0; if (createIndexEnabled) { columnModes[0] = AbstractColumn::Numeric; col = 1; } for (const auto& valueString: firstLineStringList) { // only parse columns available in first data line if (col == m_actualCols) break; columnModes[col++] = AbstractFileFilter::columnMode(valueString, dateTimeFormat, numberFormat); } QDEBUG("column modes = " << columnModes); int actualEndRow = endRow; DEBUG("endRow = " << endRow); if (endRow == -1 || endRow > m_actualRows) actualEndRow = m_actualRows; if (m_actualRows > actualEndRow) m_actualRows = actualEndRow; // reset to start of file device.seek(0); DEBUG("start/end column: " << startColumn << ' ' << endColumn); DEBUG("start/end row: " << startRow << ' ' << actualEndRow); DEBUG("actual cols/rows (w/o header incl. start rows): " << m_actualCols << ' ' << m_actualRows); if (m_actualRows == 0) return 1; return 0; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " << dataSource - << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); + DEBUG("AsciiFilterPrivate::readDataFromFile(): fileName = \'" << fileName.toStdString() << "\', dataSource = " + << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); KFilterDev device(fileName); readDataFromDevice(device, dataSource, importMode, lines); } qint64 AsciiFilterPrivate::readFromLiveDevice(QIODevice & device, AbstractDataSource * dataSource, qint64 from, AbstractFileFilter::ImportMode importMode, int lines) { - Q_ASSERT(dataSource != nullptr); - FileDataSource* spreadsheet = dynamic_cast(dataSource); - - if (!m_prepared) { - DEBUG("device is sequential = " << device.isSequential()); - const int deviceError = prepareDeviceToRead(device); - if (deviceError != 0) - DEBUG("Device error = " << deviceError); - - if (deviceError) - return 0; ////////// - - /////////////////////////// prepare import for spreadsheet - - spreadsheet->setUndoAware(false); - - //make the available columns undo unaware before we resize and rename them below, - //the same will be done for new columns in this->resize(). - for (int i = 0; i < spreadsheet->childCount(); i++) - spreadsheet->child(i)->setUndoAware(false); - - qDebug() << "fds resizing!"; - - spreadsheet->removeColumns(0, 2); - - if (importMode == AbstractFileFilter::Replace) - spreadsheet->clear(); - spreadsheet->resize(importMode, vectorNames, m_actualCols); - - qDebug() << "fds resized to col: " << m_actualCols; - - qDebug() << "fds rowCount: " << spreadsheet->rowCount(); - //also here we need a cheaper version of this - if (spreadsheet->rowCount() < m_actualRows) - spreadsheet->setRowCount(m_actualRows); - qDebug() << "fds rows resized to: " << m_actualRows; - - m_dataContainer.resize(m_actualCols); - - for (int n = 0; n < m_actualCols; n++) { - // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) - spreadsheet->child(n)->setColumnMode(columnModes[n]); - switch (columnModes[n]) { - case AbstractColumn::Numeric: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); - vector->reserve(m_actualRows); - vector->resize(m_actualRows); - m_dataContainer[n] = static_cast(vector); - break; - } - case AbstractColumn::Text: { - QVector* vector = static_cast*>(spreadsheet->child(n)->data()); - vector->reserve(m_actualRows); - vector->resize(m_actualRows); - m_dataContainer[n] = static_cast(vector); - break; - } - case AbstractColumn::DateTime: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); - vector->reserve(m_actualRows); - vector->resize(m_actualRows); - m_dataContainer[n] = static_cast(vector); - break; - } - //TODO - case AbstractColumn::Month: - case AbstractColumn::Day: - break; - } - } - - m_prepared = true; - qDebug() << "prepared!"; - } - - qint64 bytesread = 0; - - // if there's data do be read - if (device.bytesAvailable() > 0) { - - //move to the last read position, from == total bytes read - if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) { - device.seek(from); - } - - //count the new lines, increase actualrows on each - //now we read all the new lines, if we want to use sample rate - //then here we can do it, if we have actually sample rate number of lines :-? - int newLines = 0; - - while (!device.atEnd()) { - device.readLine(); - m_actualRows++; - if (spreadsheet->readingType() != FileDataSource::ReadingType::TillEnd) { - newLines++; - //for Continous reading and FromEnd we read sample rate number of lines if possible - if (newLines == spreadsheet->sampleRate()) - break; - } - } - - //back to the last read position before counting when reading from files - if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) { - device.seek(from); - } - - const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); - - //new rows - if (spreadsheet->rowCount() < m_actualRows) - spreadsheet->setRowCount(m_actualRows); - - // if we have fixed size, we do this only once in preparation, here we can use - // m_prepared and we need something to decide whether it has a fixed size or increasing - for (int n = 0; n < m_actualCols; n++) { - // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) - switch (columnModes[n]) { - case AbstractColumn::Numeric: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); - vector->reserve(m_actualRows); - vector->resize(m_actualRows); - m_dataContainer[n] = static_cast(vector); - break; - } - case AbstractColumn::Text: { - QVector* vector = static_cast*>(spreadsheet->child(n)->data()); - vector->reserve(m_actualRows); - vector->resize(m_actualRows); - m_dataContainer[n] = static_cast(vector); - break; - } - case AbstractColumn::DateTime: { - QVector* vector = static_cast* >(spreadsheet->child(n)->data()); - vector->reserve(m_actualRows); - vector->resize(m_actualRows); - m_dataContainer[n] = static_cast(vector); - break; - } - //TODO - case AbstractColumn::Month: - case AbstractColumn::Day: - break; - } - } - - // from the last row we read the new data in the spreadsheet - int currentRow = spreadsheetRowCountBeforeResize; // indexes the position in the vector(column) - - qDebug() << "reading from line: " << currentRow; - - qDebug() <<"available bytes: " << device.bytesAvailable(); - const int linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; - - qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows; - - for (int i = 0; i < /*qMin(lines, m_actualRows)*/linesToRead; ++i) { - QString line = device.readLine(); - if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) - bytesread += line.size(); - - qDebug() << "line bytes: " << line.size() << " line: " << line; - qDebug() << "reading in row: " << currentRow; - if (simplifyWhitespacesEnabled) - line = line.simplified(); - - if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines - continue; - - QLocale locale(numberFormat); - - QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); - for (int n = 0; n < m_actualCols; n++) { - if (n < lineStringList.size()) { - const QString& valueString = lineStringList.at(n); - - // set value depending on data type - switch (columnModes[n]) { - case AbstractColumn::Numeric: { - bool isNumber; - const double value = locale.toDouble(valueString, &isNumber); - static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : NAN); - qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); - break; - } - case AbstractColumn::DateTime: { - const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); - static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); - break; - } - case AbstractColumn::Text: - static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; - break; - case AbstractColumn::Month: - //TODO - break; - case AbstractColumn::Day: - //TODO - break; - } - } else { // missing columns in this line - switch (columnModes[n]) { - case AbstractColumn::Numeric: - static_cast*>(m_dataContainer[n])->operator[](currentRow) = NAN; - break; - case AbstractColumn::DateTime: - static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); - break; - case AbstractColumn::Text: - static_cast*>(m_dataContainer[n])->operator[](currentRow) = "NAN"; - break; - case AbstractColumn::Month: - //TODO - break; - case AbstractColumn::Day: - //TODO - break; - } - } - } - currentRow++; - } - - ////////// - // set the comments for each of the columns if datasource is a spreadsheet - const int rows = spreadsheet->rowCount(); - for (int n = 0; n < m_actualCols; ++n) { - Column* column = spreadsheet->column(n); - QString comment; - - switch (column->columnMode()) { - case AbstractColumn::Numeric: - comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); - break; - case AbstractColumn::Text: - comment = i18np("text data, %1 element", "text data, %1 elements", rows); - break; - } - column->setComment(comment); - - if (importMode == AbstractFileFilter::Replace) { - column->setSuppressDataChangedSignal(false); - column->setChanged(); - } - } - } - ////////////////// - return bytesread; + Q_ASSERT(dataSource != nullptr); + FileDataSource* spreadsheet = dynamic_cast(dataSource); + + if (!m_prepared) { + DEBUG("device is sequential = " << device.isSequential()); + const int deviceError = prepareDeviceToRead(device); + if (deviceError != 0) + DEBUG("Device error = " << deviceError); + + if (deviceError) + return 0; ////////// + +/////////////////////////// prepare import for spreadsheet + + spreadsheet->setUndoAware(false); + + //make the available columns undo unaware before we resize and rename them below, + //the same will be done for new columns in this->resize(). + for (int i = 0; i < spreadsheet->childCount(); i++) + spreadsheet->child(i)->setUndoAware(false); + + qDebug() << "fds resizing!"; + + spreadsheet->removeColumns(0, 2); + + if (importMode == AbstractFileFilter::Replace) + spreadsheet->clear(); + spreadsheet->resize(importMode, vectorNames, m_actualCols); + + qDebug() << "fds resized to col: " << m_actualCols; + + qDebug() << "fds rowCount: " << spreadsheet->rowCount(); + //also here we need a cheaper version of this + if (spreadsheet->rowCount() < m_actualRows) + spreadsheet->setRowCount(m_actualRows); + qDebug() << "fds rows resized to: " << m_actualRows; + + m_dataContainer.resize(m_actualCols); + + for (int n = 0; n < m_actualCols; n++) { + // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) + spreadsheet->child(n)->setColumnMode(columnModes[n]); + switch (columnModes[n]) { + case AbstractColumn::Numeric: { + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + vector->reserve(m_actualRows); + vector->resize(m_actualRows); + m_dataContainer[n] = static_cast(vector); + break; + } + case AbstractColumn::Text: { + QVector* vector = static_cast*>(spreadsheet->child(n)->data()); + vector->reserve(m_actualRows); + vector->resize(m_actualRows); + m_dataContainer[n] = static_cast(vector); + break; + } + case AbstractColumn::DateTime: { + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + vector->reserve(m_actualRows); + vector->resize(m_actualRows); + m_dataContainer[n] = static_cast(vector); + break; + } + //TODO + case AbstractColumn::Month: + case AbstractColumn::Day: + break; + } + } + + m_prepared = true; + qDebug() << "prepared!"; + } + + qint64 bytesread = 0; + + // if there's data do be read + if (device.bytesAvailable() > 0) { + + //move to the last read position, from == total bytes read + if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) + device.seek(from); + + //count the new lines, increase actualrows on each + //now we read all the new lines, if we want to use sample rate + //then here we can do it, if we have actually sample rate number of lines :-? + int newLines = 0; + + while (!device.atEnd()) { + device.readLine(); + m_actualRows++; + if (spreadsheet->readingType() != FileDataSource::ReadingType::TillEnd) { + newLines++; + //for Continous reading and FromEnd we read sample rate number of lines if possible + if (newLines == spreadsheet->sampleRate()) + break; + } + } + + //back to the last read position before counting when reading from files + if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) + device.seek(from); + + const int spreadsheetRowCountBeforeResize = spreadsheet->rowCount(); + + //new rows + if (spreadsheet->rowCount() < m_actualRows) + spreadsheet->setRowCount(m_actualRows); + + // if we have fixed size, we do this only once in preparation, here we can use + // m_prepared and we need something to decide whether it has a fixed size or increasing + for (int n = 0; n < m_actualCols; n++) { + // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) + switch (columnModes[n]) { + case AbstractColumn::Numeric: { + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + vector->reserve(m_actualRows); + vector->resize(m_actualRows); + m_dataContainer[n] = static_cast(vector); + break; + } + case AbstractColumn::Text: { + QVector* vector = static_cast*>(spreadsheet->child(n)->data()); + vector->reserve(m_actualRows); + vector->resize(m_actualRows); + m_dataContainer[n] = static_cast(vector); + break; + } + case AbstractColumn::DateTime: { + QVector* vector = static_cast* >(spreadsheet->child(n)->data()); + vector->reserve(m_actualRows); + vector->resize(m_actualRows); + m_dataContainer[n] = static_cast(vector); + break; + } + //TODO + case AbstractColumn::Month: + case AbstractColumn::Day: + break; + } + } + + // from the last row we read the new data in the spreadsheet + int currentRow = spreadsheetRowCountBeforeResize; // indexes the position in the vector(column) + + qDebug() << "reading from line: " << currentRow; + + qDebug() <<"available bytes: " << device.bytesAvailable(); + const int linesToRead = m_actualRows - spreadsheetRowCountBeforeResize; + + qDebug() << "Lines to read: " << linesToRead <<" actual rows: " << m_actualRows; + + for (int i = 0; i < /*qMin(lines, m_actualRows)*/linesToRead; ++i) { + QString line = device.readLine(); + if (spreadsheet->sourceType() == FileDataSource::SourceType::FileOrPipe) + bytesread += line.size(); + + qDebug() << "line bytes: " << line.size() << " line: " << line; + qDebug() << "reading in row: " << currentRow; + if (simplifyWhitespacesEnabled) + line = line.simplified(); + + if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines + continue; + + QLocale locale(numberFormat); + + QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); + for (int n = 0; n < m_actualCols; n++) { + if (n < lineStringList.size()) { + const QString& valueString = lineStringList.at(n); + + // set value depending on data type + switch (columnModes[n]) { + case AbstractColumn::Numeric: { + bool isNumber; + const double value = locale.toDouble(valueString, &isNumber); + static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : NAN); + qDebug() << "dataContainer[" << n << "] size:" << static_cast*>(m_dataContainer[n])->size(); + break; + } + case AbstractColumn::DateTime: { + const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); + static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); + break; + } + case AbstractColumn::Text: + static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; + break; + case AbstractColumn::Month: + //TODO + break; + case AbstractColumn::Day: + //TODO + break; + } + } else { // missing columns in this line + switch (columnModes[n]) { + case AbstractColumn::Numeric: + static_cast*>(m_dataContainer[n])->operator[](currentRow) = NAN; + break; + case AbstractColumn::DateTime: + static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); + break; + case AbstractColumn::Text: + static_cast*>(m_dataContainer[n])->operator[](currentRow) = "NAN"; + break; + case AbstractColumn::Month: + //TODO + break; + case AbstractColumn::Day: + //TODO + break; + } + } + } + currentRow++; + } + + ////////// + // set the comments for each of the columns if datasource is a spreadsheet + const int rows = spreadsheet->rowCount(); + for (int n = 0; n < m_actualCols; ++n) { + Column* column = spreadsheet->column(n); + QString comment; + + switch (column->columnMode()) { + case AbstractColumn::Numeric: + comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); + break; + case AbstractColumn::Text: + comment = i18np("text data, %1 element", "text data, %1 elements", rows); + break; + } + column->setComment(comment); + + if (importMode == AbstractFileFilter::Replace) { + column->setSuppressDataChangedSignal(false); + column->setChanged(); + } + } + } + ////////////////// + return bytesread; } /*! reads the content of device \c device to the data source \c dataSource. Uses the settings defined in the data source. */ void AsciiFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { DEBUG("AsciiFilterPrivate::readDataFromDevice(): dataSource = " << dataSource << ", mode = " << ENUM_TO_STRING(AbstractFileFilter, ImportMode, importMode) << ", lines = " << lines); Q_ASSERT(dataSource != nullptr); if (!m_prepared) { DEBUG("device is sequential = " << device.isSequential()); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) DEBUG("Device error = " << deviceError); if (deviceError == 1 && importMode == AbstractFileFilter::Replace && dataSource) dataSource->clear(); if (deviceError) return; // avoid text data in Matrix if (dynamic_cast(dataSource)) { for (auto& c: columnModes) if (c == AbstractColumn::Text) c = AbstractColumn::Numeric; } m_columnOffset = dataSource->prepareImport(m_dataContainer, importMode, m_actualRows - startRow + 1, m_actualCols, vectorNames, columnModes); m_prepared = true; } DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data int currentRow = 0; // indexes the position in the vector(column) if (lines == -1) lines = m_actualRows; DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); i++) { QString line = device.readLine(); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend the index if required //TODO: come up maybe with a solution with adding the index inside of the loop below, //without conversion to string, prepending to the list and then conversion back to integer. if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { - bool isNumber; - const double value = locale.toDouble(valueString, &isNumber); - static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : NAN); - break; - } + bool isNumber; + const double value = locale.toDouble(valueString, &isNumber); + static_cast*>(m_dataContainer[n])->operator[](currentRow) = (isNumber ? value : NAN); + break; + } case AbstractColumn::DateTime: { - const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); - static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); - break; - } + const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); + static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueDateTime.isValid() ? valueDateTime : QDateTime(); + break; + } case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else { // missing columns in this line switch (columnModes[n]) { case AbstractColumn::Numeric: static_cast*>(m_dataContainer[n])->operator[](currentRow) = NAN; break; case AbstractColumn::DateTime: static_cast*>(m_dataContainer[n])->operator[](currentRow) = QDateTime(); break; case AbstractColumn::Text: static_cast*>(m_dataContainer[n])->operator[](currentRow) = "NAN"; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } } currentRow++; emit q->completed(100 * currentRow/m_actualRows); } dataSource->finalizeImport(m_columnOffset, startColumn, endColumn, dateTimeFormat, importMode); } /*! * generates the preview for the file \c fileName reading the provided number of \c lines. */ QVector AsciiFilterPrivate::preview(const QString& fileName, int lines) { QVector dataStrings; KFilterDev device(fileName); const int deviceError = prepareDeviceToRead(device); if (deviceError != 0) { DEBUG("Device error = " << deviceError); return dataStrings; } //number formatting DEBUG("locale = " << QLocale::languageToString(numberFormat).toStdString()); QLocale locale(numberFormat); // Read the data if (lines == -1) lines = m_actualRows; DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); for (int i = 0; i < qMin(lines, m_actualRows); i++) { QString line = device.readLine(); if (simplifyWhitespacesEnabled) line = line.simplified(); if (line.isEmpty() || line.startsWith(commentCharacter)) // skip empty or commented lines continue; if (startRow > 1) { // skip start lines startRow--; continue; } QStringList lineStringList = line.split(m_separator, QString::SkipEmptyParts); //prepend index if required if (createIndexEnabled) lineStringList.prepend(QString::number(i+1)); QStringList lineString; for (int n = 0; n < m_actualCols; n++) { if (n < lineStringList.size()) { const QString& valueString = lineStringList.at(n); // set value depending on data type switch (columnModes[n]) { case AbstractColumn::Numeric: { - bool isNumber; - const double value = locale.toDouble(valueString, &isNumber); - lineString += QString::number(isNumber ? value : NAN); - break; - } + bool isNumber; + const double value = locale.toDouble(valueString, &isNumber); + lineString += QString::number(isNumber ? value : NAN); + break; + } case AbstractColumn::DateTime: { - const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); - lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); - break; - } + const QDateTime valueDateTime = QDateTime::fromString(valueString, dateTimeFormat); + lineString += valueDateTime.isValid() ? valueDateTime.toString(dateTimeFormat) : QLatin1String(" "); + break; + } case AbstractColumn::Text: lineString += valueString; break; case AbstractColumn::Month: //TODO break; case AbstractColumn::Day: //TODO break; } } else // missing columns in this line lineString += QLatin1String("NAN"); } dataStrings << lineString; } return dataStrings; } /*! writes the content of \c dataSource to the file \c fileName. */ void AsciiFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void AsciiFilter::save(QXmlStreamWriter* writer) const { - writer->writeStartElement( "asciiFilter"); - writer->writeAttribute( "commentCharacter", d->commentCharacter); - writer->writeAttribute( "separatingCharacter", d->separatingCharacter); - writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); - writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); - writer->writeAttribute( "header", QString::number(d->headerEnabled)); + writer->writeStartElement( "asciiFilter"); + writer->writeAttribute( "commentCharacter", d->commentCharacter); + writer->writeAttribute( "separatingCharacter", d->separatingCharacter); + writer->writeAttribute( "autoMode", QString::number(d->autoModeEnabled)); + writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled)); + writer->writeAttribute( "header", QString::number(d->headerEnabled)); writer->writeAttribute( "vectorNames", d->vectorNames.join(' ')); - writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); - writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); - writer->writeAttribute( "startRow", QString::number(d->startRow)); - writer->writeAttribute( "endRow", QString::number(d->endRow)); - writer->writeAttribute( "startColumn", QString::number(d->startColumn)); - writer->writeAttribute( "endColumn", QString::number(d->endColumn)); + writer->writeAttribute( "skipEmptyParts", QString::number(d->skipEmptyParts)); + writer->writeAttribute( "simplifyWhitespaces", QString::number(d->simplifyWhitespacesEnabled)); + writer->writeAttribute( "startRow", QString::number(d->startRow)); + writer->writeAttribute( "endRow", QString::number(d->endRow)); + writer->writeAttribute( "startColumn", QString::number(d->startColumn)); + writer->writeAttribute( "endColumn", QString::number(d->endColumn)); writer->writeEndElement(); } /*! Loads from XML. */ bool AsciiFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "asciiFilter") { reader->raiseError(i18n("no ascii filter element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); QString str = attribs.value("commentCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'commentCharacter'")); else d->commentCharacter = str; str = attribs.value("separatingCharacter").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'separatingCharacter'")); else d->separatingCharacter = str; str = attribs.value("createIndex").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'createIndex'")); else d->createIndexEnabled = str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'autoMode'")); else d->autoModeEnabled = str.toInt(); str = attribs.value("header").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'header'")); else d->headerEnabled = str.toInt(); str = attribs.value("vectorNames").toString(); d->vectorNames = str.split(' '); //may be empty str = attribs.value("simplifyWhitespaces").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'simplifyWhitespaces'")); else d->simplifyWhitespacesEnabled = str.toInt(); str = attribs.value("skipEmptyParts").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipEmptyParts'")); else d->skipEmptyParts = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("startColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startColumn'")); else d->startColumn = str.toInt(); str = attribs.value("endColumn").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endColumn'")); else d->endColumn = str.toInt(); return true; } diff --git a/src/backend/datasources/filters/AsciiFilter.h b/src/backend/datasources/filters/AsciiFilter.h index b61180fb8..556004504 100644 --- a/src/backend/datasources/filters/AsciiFilter.h +++ b/src/backend/datasources/filters/AsciiFilter.h @@ -1,111 +1,111 @@ /*************************************************************************** File : AsciiFilter.h Project : LabPlot Description : ASCII I/O-filter -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTER_H #define ASCIIFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/core/AbstractColumn.h" class QStringList; class QIODevice; class AsciiFilterPrivate; class AsciiFilter : public AbstractFileFilter { Q_OBJECT public: AsciiFilter(); ~AsciiFilter(); static QStringList separatorCharacters(); static QStringList commentCharacters(); static QStringList dataTypes(); static QStringList predefinedFilters(); static int columnNumber(const QString& fileName, const QString& separator = QString()); static size_t lineNumber(const QString& fileName); static size_t lineNumber(QIODevice&); // calculate number of lines if device supports it // read data from any device - void readDataFromDevice(QIODevice& device, AbstractDataSource*, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); - void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*dataSource, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); - qint64 readFromLiveDevice(QIODevice& device, AbstractDataSource*, - qint64 from = -1, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + void readDataFromDevice(QIODevice& device, AbstractDataSource*, + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*dataSource, + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); + qint64 readFromLiveDevice(QIODevice& device, AbstractDataSource*, + qint64 from = -1, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); // overloaded function to read from file QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); void loadFilterSettings(const QString&); void saveFilterSettings(const QString&) const; void setCommentCharacter(const QString&); QString commentCharacter() const; void setSeparatingCharacter(const QString&); QString separatingCharacter() const; void setDateTimeFormat(const QString&); QString dateTimeFormat() const; void setNumberFormat(QLocale::Language); QLocale::Language numberFormat() const; void setAutoModeEnabled(const bool); bool isAutoModeEnabled() const; void setHeaderEnabled(const bool); bool isHeaderEnabled() const; void setSkipEmptyParts(const bool); bool skipEmptyParts() const; void setSimplifyWhitespacesEnabled(const bool); bool simplifyWhitespacesEnabled() const; void setCreateIndexEnabled(const bool); void setVectorNames(const QString); QStringList vectorNames() const; QVector columnModes(); void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; void setStartColumn(const int); int startColumn() const; void setEndColumn(const int); int endColumn() const; virtual void save(QXmlStreamWriter*) const; virtual bool load(XmlStreamReader*); private: std::unique_ptr const d; friend class AsciiFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/AsciiFilterPrivate.h b/src/backend/datasources/filters/AsciiFilterPrivate.h index f19d9fffe..a7ed55e3d 100644 --- a/src/backend/datasources/filters/AsciiFilterPrivate.h +++ b/src/backend/datasources/filters/AsciiFilterPrivate.h @@ -1,85 +1,85 @@ /*************************************************************************** File : AsciiFilterPrivate.h Project : LabPlot Description : Private implementation class for AsciiFilter. -------------------------------------------------------------------- Copyright : (C) 2009-2013 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef ASCIIFILTERPRIVATE_H #define ASCIIFILTERPRIVATE_H #include class KFilterDev; class AbstractDataSource; class AbstractColumn; class AsciiFilterPrivate { public: explicit AsciiFilterPrivate(AsciiFilter*); int prepareDeviceToRead(QIODevice&); void readDataFromDevice(QIODevice&, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readFromLiveDeviceNotFile(QIODevice& device, AbstractDataSource*, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace); qint64 readFromLiveDevice(QIODevice&, AbstractDataSource*, qint64 from = -1, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, - AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); + AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); QVector preview(const QString& fileName, int lines); const AsciiFilter* q; QString commentCharacter; QString separatingCharacter; QString dateTimeFormat; QLocale::Language numberFormat; - bool createIndexEnabled; bool autoModeEnabled; bool headerEnabled; bool skipEmptyParts; bool simplifyWhitespacesEnabled; + bool createIndexEnabled; QStringList vectorNames; QVector columnModes; int startRow; int endRow; int startColumn; int endColumn; private: QString m_separator; int m_actualRows; int m_actualCols; int m_prepared; int m_columnOffset; // indexes the "start column" in the datasource. Data will be imported starting from this column. QVector m_dataContainer; // pointers to the actual data containers void clearDataSource(AbstractDataSource*) const; }; #endif diff --git a/src/backend/datasources/filters/BinaryFilter.cpp b/src/backend/datasources/filters/BinaryFilter.cpp index d08bebb66..740619846 100644 --- a/src/backend/datasources/filters/BinaryFilter.cpp +++ b/src/backend/datasources/filters/BinaryFilter.cpp @@ -1,499 +1,617 @@ /*************************************************************************** File : BinaryFilter.cpp Project : LabPlot Description : Binary I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/BinaryFilter.h" #include "backend/datasources/filters/BinaryFilterPrivate.h" #include "backend/datasources/FileDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include /*! \class BinaryFilter \brief Manages the import/export of data organized as columns (vectors) from/to a binary file. \ingroup datasources */ BinaryFilter::BinaryFilter():AbstractFileFilter(), d(new BinaryFilterPrivate(this)) {} BinaryFilter::~BinaryFilter() {} /*! reads the content of the file \c fileName. */ QVector BinaryFilter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - return d->readDataFromFile(fileName, dataSource, importMode, lines); + d->readDataFromFile(fileName, dataSource, importMode, lines); + return QVector(); //TODO: remove this later once all read*-functions in the filter classes don't return any preview strings anymore } /*! reads the content of the device \c device. */ -QVector BinaryFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - return d->readDataFromDevice(device, dataSource, importMode, lines); +void BinaryFilter::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { + d->readDataFromDevice(device, dataSource, importMode, lines); +} + +QVector BinaryFilter::preview(const QString& fileName, int lines) { + return d->preview(fileName, lines); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void BinaryFilter::write(const QString & fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); // emit() } /*! returns the list of all predefined data formats. */ QStringList BinaryFilter::dataTypes() { return (QStringList()<<"int8 (8 bit signed integer)"<<"int16 (16 bit signed integer)"<<"int32 (32 bit signed integer)"<<"int64 (64 bit signed integer)" <<"uint8 (8 bit unsigned integer)"<<"uint16 (16 bit unsigned integer)"<<"uint32 (32 bit unsigned integer)"<<"uint64 (64 bit unsigned integer)" <<"real32 (single precision floats)"<<"real64 (double precision floats)"); } /*! returns the list of all predefined byte order. */ QStringList BinaryFilter::byteOrders() { - return (QStringList()<<"Little endian"<<"Big endian"); + return (QStringList() << "Little endian" << "Big endian"); } /*! returns the size of the predefined data types */ int BinaryFilter::dataSize(BinaryFilter::DataType type) { - int sizes[]={1,2,4,8,1,2,4,8,4,8}; + int sizes[] = {1,2,4,8,1,2,4,8,4,8}; return sizes[(int)type]; } /*! returns the number of rows (length of vectors) in the file \c fileName. */ -long BinaryFilter::rowNumber(const QString & fileName, const int vectors, const BinaryFilter::DataType type) { +size_t BinaryFilter::rowNumber(const QString& fileName, const int vectors, const BinaryFilter::DataType type) { KFilterDev device(fileName); if (!device.open(QIODevice::ReadOnly)) return 0; - long rows=0; + size_t rows = 0; while (!device.atEnd()) { // one row - for (int i=0; i < vectors; ++i){ - for (int j=0; j < BinaryFilter::dataSize(type); ++j) + for (int i = 0; i < vectors; ++i){ + for (int j = 0; j < BinaryFilter::dataSize(type); ++j) device.read(1); } rows++; } return rows; } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void BinaryFilter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void BinaryFilter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void BinaryFilter::setVectors(const int v) { d->vectors = v; } int BinaryFilter::vectors() const { return d->vectors; } void BinaryFilter::setDataType(const BinaryFilter::DataType t) { d->dataType = t; } BinaryFilter::DataType BinaryFilter::dataType() const { return d->dataType; } void BinaryFilter::setByteOrder(const BinaryFilter::ByteOrder b) { d->byteOrder = b; } BinaryFilter::ByteOrder BinaryFilter::byteOrder() const{ return d->byteOrder; } void BinaryFilter::setSkipStartBytes(const int s) { d->skipStartBytes = s; } int BinaryFilter::skipStartBytes() const { return d->skipStartBytes; } void BinaryFilter::setStartRow(const int s) { d->startRow = s; } int BinaryFilter::startRow() const { return d->startRow; } void BinaryFilter::setEndRow(const int e) { d->endRow = e; } int BinaryFilter::endRow() const { return d->endRow; } void BinaryFilter::setSkipBytes(const int s) { d->skipBytes = s; } int BinaryFilter::skipBytes() const { return d->skipBytes; } +void BinaryFilter::setCreateIndexEnabled(bool b) { + d->createIndexEnabled = b; +} + void BinaryFilter::setAutoModeEnabled(bool b) { d->autoModeEnabled = b; } bool BinaryFilter::isAutoModeEnabled() const { return d->autoModeEnabled; } //##################################################################### //################### Private implementation ########################## //##################################################################### BinaryFilterPrivate::BinaryFilterPrivate(BinaryFilter* owner) : q(owner), vectors(2), dataType(BinaryFilter::INT8), byteOrder(BinaryFilter::LittleEndian), - skipStartBytes(0), startRow(1), endRow(-1), + skipStartBytes(0), skipBytes(0), + createIndexEnabled(false), autoModeEnabled(true) { } /*! reads the content of the device \c device to the data source \c dataSource or return as string for preview. Uses the settings defined in the data source. */ -QVector BinaryFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - QVector dataStrings; +void BinaryFilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { + DEBUG("readDataFromFile()"); KFilterDev device(fileName); - if (! device.open(QIODevice::ReadOnly)) - return dataStrings << (QStringList() << i18n("could not open device")); - numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); - return readDataFromDevice(device, dataSource, importMode, lines); + if (! device.open(QIODevice::ReadOnly)) { + DEBUG(" could not open file " << fileName.toStdString()); + return; + } + readDataFromDevice(device, dataSource, importMode, lines); } -/*! - reads the content of the file \c fileName to the data source \c dataSource or return as string for preview. - Uses the settings defined in the data source. -*/ -QVector BinaryFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { - QVector dataStrings; - QDataStream in(&device); +/*! + * returns 1 if the current read position in the device is at the end and 0 otherwise. + */ +int BinaryFilterPrivate::prepareStreamToRead(QDataStream& in) { + DEBUG("prepareStreamToRead()"); - //TODO: check if (byteOrder == BinaryFilter::BigEndian) in.setByteOrder(QDataStream::BigEndian); else if (byteOrder == BinaryFilter::LittleEndian) in.setByteOrder(QDataStream::LittleEndian); // catch case that skipStartBytes or startRow is bigger than file - if (skipStartBytes >= BinaryFilter::dataSize(dataType) * vectors * numRows || startRow > numRows) { - if (dataSource) - dataSource->clear(); - return dataStrings << (QStringList() << i18n("data selection empty")); - } + if (skipStartBytes >= BinaryFilter::dataSize(dataType) * vectors * numRows || startRow > numRows) + return 1; // skip bytes at start for (int i = 0; i < skipStartBytes; i++) { qint8 tmp; in >> tmp; } // skip until start row for (int i = 0; i < (startRow-1) * vectors; ++i) { for (int j = 0; j < BinaryFilter::dataSize(dataType); ++j) { qint8 tmp; in >> tmp; } } // set range of rows - int actualRows; if (endRow == -1) - actualRows = numRows - startRow + 1; + m_actualRows = numRows - startRow + 1; else if (endRow > numRows - startRow + 1) - actualRows = numRows; + m_actualRows = numRows; else - actualRows = endRow - startRow + 1; - int actualCols = vectors; - if (lines == -1) - lines = actualRows; - - //TODO: use DEBUG() -#ifndef NDEBUG - qDebug()<<" numRows ="< dataStrings; + + KFilterDev device(fileName); + if (! device.open(QIODevice::ReadOnly)) + return dataStrings << (QStringList() << i18n("could not open device")); + + numRows = BinaryFilter::rowNumber(fileName, vectors, dataType); + + QDataStream in(&device); + const int deviceError = prepareStreamToRead(in); + + if(deviceError) + return dataStrings << (QStringList() << i18n("data selection empty")); //TODO: support other modes QVector columnModes; - columnModes.resize(actualCols); + columnModes.resize(m_actualCols); - //TODO: use given names? + //TODO: use given names QStringList vectorNames; - if (dataSource) - columnOffset = dataSource->prepareImport(dataContainer, importMode, actualRows, actualCols, vectorNames, columnModes); + if (createIndexEnabled) + vectorNames.prepend("index"); + + if (lines == -1) + lines = m_actualRows; // read data //TODO: use ColumnMode ? - for (int i = 0; i < qMin(actualRows, lines); i++) { + DEBUG("generating preview for " << qMin(lines, m_actualRows) << " lines"); + for (int i = 0; i < qMin(m_actualRows, lines); i++) { QStringList lineString; - for (int n = 0; n < actualCols; n++) { + + //prepend the index if required + if (createIndexEnabled) + lineString << QString::number(i+1); + + for (int n = 0; n < m_actualCols; n++) { switch (dataType) { case BinaryFilter::INT8: { qint8 value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::INT16: { qint16 value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::INT32: { qint32 value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::INT64: { qint64 value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::UINT8: { quint8 value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::UINT16: { quint16 value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::UINT32: { quint32 value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::UINT64: { quint64 value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::REAL32: { float value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } case BinaryFilter::REAL64: { double value; in >> value; - if (dataSource) - static_cast*>(dataContainer[n])->operator[](i) = value; - else - lineString << QString::number(value); + lineString << QString::number(value); break; } } } dataStrings << lineString; - emit q->completed(100*i/actualRows); + emit q->completed(100*i/m_actualRows); } - if (!dataSource) - return dataStrings; - - dataSource->finalizeImport(columnOffset, 1, actualCols, "", importMode); return dataStrings; } + /*! + reads the content of the file \c fileName to the data source \c dataSource or return as string for preview. + Uses the settings defined in the data source. +*/ +void BinaryFilterPrivate::readDataFromDevice(QIODevice& device, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode importMode, int lines) { + DEBUG("BinaryFilterPrivate::readDataFromDevice()"); + + QDataStream in(&device); + const int deviceError = prepareStreamToRead(in); + + if (deviceError) { + dataSource->clear(); + DEBUG("device error"); + return; + } + + if (createIndexEnabled) + m_actualCols++; + + QVector dataContainer; + int columnOffset = 0; + + //TODO: support other modes + QVector columnModes; + columnModes.resize(m_actualCols); + + //TODO: use given names + QStringList vectorNames; + + if (createIndexEnabled) + vectorNames.prepend("index"); + + columnOffset = dataSource->prepareImport(dataContainer, importMode, m_actualRows, m_actualCols, vectorNames, columnModes); + + if (lines == -1) + lines = m_actualRows; + + // start column + int startColumn = 0; + if (createIndexEnabled) + startColumn++; + + // read data + //TODO: use ColumnMode ? + DEBUG("reading " << qMin(lines, m_actualRows) << " lines"); + for (int i = 0; i < qMin(m_actualRows, lines); i++) { + //prepend the index if required + if (createIndexEnabled) + static_cast*>(dataContainer[0])->operator[](i) = i+1; + + for (int n = startColumn; n < m_actualCols; n++) { + switch (dataType) { + case BinaryFilter::INT8: { + qint8 value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::INT16: { + qint16 value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::INT32: { + qint32 value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::INT64: { + qint64 value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::UINT8: { + quint8 value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::UINT16: { + quint16 value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::UINT32: { + quint32 value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::UINT64: { + quint64 value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::REAL32: { + float value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + case BinaryFilter::REAL64: { + double value; + in >> value; + static_cast*>(dataContainer[n])->operator[](i) = value; + break; + } + } + } + emit q->completed(100*i/m_actualRows); + } + + dataSource->finalizeImport(columnOffset, 1, m_actualCols, "", importMode); +} + /*! writes the content of \c dataSource to the file \c fileName. */ void BinaryFilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void BinaryFilter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("binaryFilter"); writer->writeAttribute("vectors", QString::number(d->vectors) ); writer->writeAttribute("dataType", QString::number(d->dataType) ); writer->writeAttribute("byteOrder", QString::number(d->byteOrder) ); writer->writeAttribute("autoMode", QString::number(d->autoModeEnabled) ); writer->writeAttribute("startRow", QString::number(d->startRow) ); writer->writeAttribute("endRow", QString::number(d->endRow) ); writer->writeAttribute("skipStartBytes", QString::number(d->skipStartBytes) ); writer->writeAttribute("skipBytes", QString::number(d->skipBytes) ); + writer->writeAttribute( "createIndex", QString::number(d->createIndexEnabled) ); writer->writeEndElement(); } /*! Loads from XML. */ bool BinaryFilter::load(XmlStreamReader* reader) { if (!reader->isStartElement() || reader->name() != "binaryFilter") { reader->raiseError(i18n("no binary filter element found")); return false; } QString attributeWarning = i18n("Attribute '%1' missing or empty, default value is used"); QXmlStreamAttributes attribs = reader->attributes(); // read attributes QString str = attribs.value("vectors").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'vectors'")); else d->vectors = str.toInt(); str = attribs.value("dataType").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'dataType'")); else d->dataType = (BinaryFilter::DataType) str.toInt(); str = attribs.value("byteOrder").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'byteOrder'")); else d->byteOrder = (BinaryFilter::ByteOrder) str.toInt(); str = attribs.value("autoMode").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'autoMode'")); else d->autoModeEnabled = str.toInt(); str = attribs.value("startRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'startRow'")); else d->startRow = str.toInt(); str = attribs.value("endRow").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'endRow'")); else d->endRow = str.toInt(); str = attribs.value("skipStartBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipStartBytes'")); else d->skipStartBytes = str.toInt(); str = attribs.value("skipBytes").toString(); if (str.isEmpty()) reader->raiseWarning(attributeWarning.arg("'skipBytes'")); else d->skipBytes = str.toInt(); + str = attribs.value("createIndex").toString(); + if (str.isEmpty()) + reader->raiseWarning(attributeWarning.arg("'createIndex'")); + else + d->createIndexEnabled = str.toInt(); + return true; } diff --git a/src/backend/datasources/filters/BinaryFilter.h b/src/backend/datasources/filters/BinaryFilter.h index fad646f17..28cdabb31 100644 --- a/src/backend/datasources/filters/BinaryFilter.h +++ b/src/backend/datasources/filters/BinaryFilter.h @@ -1,95 +1,93 @@ /*************************************************************************** File : BinaryFilter.h Project : LabPlot Description : Binary I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef BINARYFILTER_H #define BINARYFILTER_H #include "backend/datasources/filters/AbstractFileFilter.h" class BinaryFilterPrivate; class QStringList; class QIODevice; class BinaryFilter : public AbstractFileFilter { Q_OBJECT Q_ENUMS(DataType) Q_ENUMS(ByteOrder) public: //TODO; use ColumnMode? enum DataType {INT8, INT16, INT32, INT64, UINT8, UINT16, UINT32, UINT64, REAL32, REAL64}; enum ByteOrder {LittleEndian, BigEndian}; BinaryFilter(); ~BinaryFilter(); static QStringList dataTypes(); static QStringList byteOrders(); static int dataSize(BinaryFilter::DataType); - static long rowNumber(const QString& fileName, const int vectors, const BinaryFilter::DataType); + static size_t rowNumber(const QString& fileName, const int vectors, const BinaryFilter::DataType); // read data from any device - QVector readDataFromDevice(QIODevice&, AbstractDataSource* = nullptr, + void readDataFromDevice(QIODevice&, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); - QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, + QVector readDataFromFile(const QString& fileName, AbstractDataSource*, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); + QVector preview(const QString& fileName, int lines); void loadFilterSettings(const QString&); void saveFilterSettings(const QString&) const; void setVectors(const int); int vectors() const; void setDataType(const BinaryFilter::DataType); BinaryFilter::DataType dataType() const; - void setByteOrder(const BinaryFilter::ByteOrder); BinaryFilter::ByteOrder byteOrder() const; - void setSkipStartBytes(const int); int skipStartBytes() const; - void setStartRow(const int); int startRow() const; void setEndRow(const int); int endRow() const; - void setSkipBytes(const int); int skipBytes() const; + void setCreateIndexEnabled(const bool); void setAutoModeEnabled(const bool); bool isAutoModeEnabled() const; virtual void save(QXmlStreamWriter*) const; virtual bool load(XmlStreamReader*); private: std::unique_ptr const d; friend class BinaryFilterPrivate; }; #endif diff --git a/src/backend/datasources/filters/BinaryFilterPrivate.h b/src/backend/datasources/filters/BinaryFilterPrivate.h index 6f23fbb71..9df0ea701 100644 --- a/src/backend/datasources/filters/BinaryFilterPrivate.h +++ b/src/backend/datasources/filters/BinaryFilterPrivate.h @@ -1,61 +1,67 @@ /*************************************************************************** File : BinaryFilterPrivate.h Project : LabPlot Description : Private implementation class for BinaryFilter. -------------------------------------------------------------------- Copyright : (C) 2015-2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef BINARYFILTERPRIVATE_H #define BINARYFILTERPRIVATE_H class AbstractDataSource; class BinaryFilterPrivate { public: explicit BinaryFilterPrivate(BinaryFilter*); - QVector readDataFromDevice(QIODevice& device, AbstractDataSource* = nullptr, + int prepareStreamToRead(QDataStream&); + void readDataFromDevice(QIODevice& device, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); - QVector readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, + void readDataFromFile(const QString& fileName, AbstractDataSource* = nullptr, AbstractFileFilter::ImportMode = AbstractFileFilter::Replace, int lines = -1); void write(const QString& fileName, AbstractDataSource*); + QVector preview(const QString& fileName, int lines); const BinaryFilter* q; int vectors; BinaryFilter::DataType dataType; BinaryFilter::ByteOrder byteOrder; - int skipStartBytes; // bytes to skip at start int startRow; // start row (value*vectors) to read - int endRow; // end row to (value*vectors) read - int skipBytes; // bytes to skip after each value + int endRow; // end row to (value*vectors) read int numRows; // number of rows + int skipStartBytes; // bytes to skip at start + int skipBytes; // bytes to skip after each value + bool createIndexEnabled; // if create index column bool autoModeEnabled; private: void clearDataSource(AbstractDataSource*) const; + + int m_actualRows; + int m_actualCols; }; #endif diff --git a/src/backend/spreadsheet/Spreadsheet.cpp b/src/backend/spreadsheet/Spreadsheet.cpp index 1d84a61c8..c64f80cb5 100644 --- a/src/backend/spreadsheet/Spreadsheet.cpp +++ b/src/backend/spreadsheet/Spreadsheet.cpp @@ -1,937 +1,937 @@ /*************************************************************************** File : Spreadsheet.cpp Project : LabPlot Description : Aspect providing a spreadsheet table with column logic -------------------------------------------------------------------- Copyright : (C) 2012-2016 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2006-2008 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2006-2009 Knut Franke (knut.franke@gmx.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "Spreadsheet.h" #include "backend/core/AspectPrivate.h" #include "backend/core/AbstractAspect.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/datatypes/DateTime2StringFilter.h" #include "commonfrontend/spreadsheet/SpreadsheetView.h" #include "kdefrontend/spreadsheet/ExportSpreadsheetDialog.h" #include #include #include #include #include #include /*! \class Spreadsheet \brief Aspect providing a spreadsheet table with column logic. Spreadsheet is a container object for columns with no data of its own. By definition, it's columns are all of its children inheriting from class Column. Thus, the basic API is already defined by AbstractAspect (managing the list of columns, notification of column insertion/removal) and Column (changing and monitoring state of the actual data). Spreadsheet stores a pointer to its primary view of class SpreadsheetView. SpreadsheetView calls the Spreadsheet API but Spreadsheet only notifies SpreadsheetView by signals without calling its API directly. This ensures a maximum independence of UI and backend. SpreadsheetView can be easily replaced by a different class. User interaction is completely handled in SpreadsheetView and translated into Spreadsheet API calls (e.g., when a user edits a cell this will be handled by the delegate of SpreadsheetView and Spreadsheet will not know whether a script or a user changed the data.). All actions, menus etc. for the user interaction are handled SpreadsheetView, e.g., via a context menu. Selections are also handled by SpreadsheetView. The view itself is created by the first call to view(); \ingroup backend */ Spreadsheet::Spreadsheet(AbstractScriptingEngine* engine, const QString& name, bool loading) : AbstractDataSource(engine, name) { if (!loading) init(); } /*! initializes the spreadsheet with the default number of columns and rows */ void Spreadsheet::init() { int columns, rows; KConfig config; KConfigGroup group = config.group( "Spreadsheet" ); columns = group.readEntry("ColumnCount", 2); rows = group.readEntry("RowCount", 100); for (int i = 0; i < columns; i++) { Column* new_col = new Column(QString::number(i+1), AbstractColumn::Numeric); new_col->setPlotDesignation(i == 0 ? AbstractColumn::X : AbstractColumn::Y); addChild(new_col); } setRowCount(rows); } /*! Constructs a primary view on me. This method may be called multiple times during the life time of an Aspect, or it might not get called at all. Aspects must not depend on the existence of a view for their operation. */ QWidget *Spreadsheet::view() const { if (!m_view) { m_view = new SpreadsheetView(const_cast(this)); KConfig config; KConfigGroup group = config.group( "Spreadsheet" ); reinterpret_cast(m_view)->showComments(group.readEntry("ShowComments", false)); } return m_view; } bool Spreadsheet::exportView() const { ExportSpreadsheetDialog* dlg = new ExportSpreadsheetDialog(view()); dlg->setFileName(name()); dlg->setExportTo(QStringList() << i18n("FITS image") << i18n("FITS table")); for (int i = 0; i < columnCount(); ++i) { if (column(i)->columnMode() != AbstractColumn::Numeric) { dlg->setExportToImage(false); break; } } if (const_cast(reinterpret_cast(m_view))->selectedColumnCount() == 0) { dlg->setExportSelection(false); } bool ret; if ((ret = dlg->exec()) == QDialog::Accepted) { const QString path = dlg->path(); const bool exportHeader = dlg->exportHeader(); const SpreadsheetView* view = reinterpret_cast(m_view); WAIT_CURSOR; if (dlg->format() == ExportSpreadsheetDialog::LaTeX) { const bool exportLatexHeader = dlg->exportLatexHeader(); const bool gridLines = dlg->gridLines(); const bool captions = dlg->captions(); const bool skipEmptyRows =dlg->skipEmptyRows(); const bool exportEntire = dlg->entireSpreadheet(); view->exportToLaTeX(path, exportHeader, gridLines, captions, exportLatexHeader, skipEmptyRows, exportEntire); } else if (dlg->format() == ExportSpreadsheetDialog::FITS) { const int exportTo = dlg->exportToFits(); const bool commentsAsUnits = dlg->commentsAsUnitsFits(); view->exportToFits(path, exportTo, commentsAsUnits); } else { const QString separator = dlg->separator(); view->exportToFile(path, exportHeader, separator); } RESET_CURSOR; } delete dlg; return ret; } bool Spreadsheet::printView() { QPrinter printer; QPrintDialog* dlg = new QPrintDialog(&printer, view()); bool ret; dlg->setWindowTitle(i18n("Print Spreadsheet")); if ((ret = dlg->exec()) == QDialog::Accepted) { const SpreadsheetView* view = reinterpret_cast(m_view); view->print(&printer); } delete dlg; return ret; } bool Spreadsheet::printPreview() const { const SpreadsheetView* view = reinterpret_cast(m_view); QPrintPreviewDialog* dlg = new QPrintPreviewDialog(m_view); connect(dlg, SIGNAL(paintRequested(QPrinter*)), view, SLOT(print(QPrinter*))); return dlg->exec(); } /*! Returns the total number of rows in the spreadsheet. */ int Spreadsheet::rowCount() const { int col_rows, result = 0; for (auto* col: children()) if ((col_rows = col->rowCount()) > result) result = col_rows; return result; } void Spreadsheet::removeRows(int first, int count) { if( count < 1 || first < 0 || first+count > rowCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: remove 1 row", "%1: remove %2 rows", name(), count) ); for (auto* col: children(IncludeHidden)) col->removeRows(first, count); endMacro(); RESET_CURSOR; } void Spreadsheet::insertRows(int before, int count) { if( count < 1 || before < 0 || before > rowCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: insert 1 row", "%1: insert %2 rows", name(), count) ); for (auto* col: children(IncludeHidden)) col->insertRows(before, count); endMacro(); RESET_CURSOR; } void Spreadsheet::appendRows(int count){ insertRows(rowCount(), count); } void Spreadsheet::appendRow(){ insertRows(rowCount(), 1); } void Spreadsheet::appendColumns(int count){ insertColumns(columnCount(), count); } void Spreadsheet::appendColumn(){ insertColumns(columnCount(), 1); } void Spreadsheet::prependColumns(int count){ insertColumns(0, count); } /*! Sets the number of rows of the spreadsheet to \c new_size */ void Spreadsheet::setRowCount(int new_size) { int current_size = rowCount(); if (new_size > current_size) insertRows(current_size, new_size-current_size); if (new_size < current_size && new_size >= 0) removeRows(new_size, current_size-new_size); } /*! Returns the column with the number \c index. Shallow wrapper around \sa AbstractAspect::child() - see there for caveat. */ Column* Spreadsheet::column(int index) const{ return child(index); } /*! Returns the column with the name \c name. */ Column* Spreadsheet::column(const QString &name) const{ return child(name); } /*! Returns the total number of columns in the spreadsheet. */ int Spreadsheet::columnCount() const{ return childCount(); } /*! Returns the number of columns matching the given designation. */ int Spreadsheet::columnCount(AbstractColumn::PlotDesignation pd) const { int count = 0; for (auto* col: children()) if (col->plotDesignation() == pd) count++; return count; } void Spreadsheet::removeColumns(int first, int count) { if( count < 1 || first < 0 || first+count > columnCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: remove 1 column", "%1: remove %2 columns", name(), count) ); for (int i = 0; i < count; i++) child(first)->remove(); endMacro(); RESET_CURSOR; } void Spreadsheet::insertColumns(int before, int count) { WAIT_CURSOR; beginMacro( i18np("%1: insert 1 column", "%1: insert %2 columns", name(), count) ); Column * before_col = column(before); int rows = rowCount(); for (int i = 0; i < count; i++) { Column * new_col = new Column(QString::number(i+1), AbstractColumn::Numeric); new_col->setPlotDesignation(AbstractColumn::Y); new_col->insertRows(0, rows); insertChildBefore(new_col, before_col); } endMacro(); RESET_CURSOR; } /*! Sets the number of columns to \c new_size */ void Spreadsheet::setColumnCount(int new_size) { int old_size = columnCount(); if ( old_size == new_size || new_size < 0 ) return; if (new_size < old_size) removeColumns(new_size, old_size-new_size); else insertColumns(old_size, new_size-old_size); } /*! Clears the whole spreadsheet. */ void Spreadsheet::clear() { WAIT_CURSOR; beginMacro(i18n("%1: clear", name())); for (auto* col: children()) col->clear(); endMacro(); RESET_CURSOR; } /*! Clears all mask in the spreadsheet. */ void Spreadsheet::clearMasks() { WAIT_CURSOR; beginMacro(i18n("%1: clear all masks", name())); for (auto* col: children()) col->clearMasks(); endMacro(); RESET_CURSOR; } /*! Returns a new context menu. The caller takes ownership of the menu. */ QMenu *Spreadsheet::createContextMenu(){ QMenu *menu = AbstractPart::createContextMenu(); Q_ASSERT(menu); emit requestProjectContextMenu(menu); return menu; } void Spreadsheet::moveColumn(int from, int to) { Column * col = child(from); beginMacro(i18n("%1: move column %2 from position %3 to %4.", name(), col->name(), from+1, to+1)); col->remove(); insertChildBefore(col, child(to)); endMacro(); } void Spreadsheet::copy(Spreadsheet * other) { WAIT_CURSOR; beginMacro(i18n("%1: copy %2", name(), other->name())); for (auto* col: children()) col->remove(); for (auto* src_col: other->children()) { Column * new_col = new Column(src_col->name(), src_col->columnMode()); new_col->copy(src_col); new_col->setPlotDesignation(src_col->plotDesignation()); QList< Interval > masks = src_col->maskedIntervals(); for (const auto& iv: masks) new_col->setMasked(iv); QList< Interval > formulas = src_col->formulaIntervals(); for (const auto& iv: formulas) new_col->setFormula(iv, src_col->formula(iv.start())); new_col->setWidth(src_col->width()); addChild(new_col); } setComment(other->comment()); endMacro(); RESET_CURSOR; } // FIXME: replace index-based API with Column*-based one /*! Determines the corresponding X column. */ int Spreadsheet::colX(int col) { for(int i = col-1; i >= 0; i--) { if (column(i)->plotDesignation() == AbstractColumn::X) return i; } int cols = columnCount(); for(int i = col+1; i < cols; i++) { if (column(i)->plotDesignation() == AbstractColumn::X) return i; } return -1; } /*! Determines the corresponding Y column. */ int Spreadsheet::colY(int col) { int cols = columnCount(); if (column(col)->plotDesignation() == AbstractColumn::XError || column(col)->plotDesignation() == AbstractColumn::YError) { // look to the left first for(int i=col-1; i>=0; i--) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } for(int i=col+1; iplotDesignation() == AbstractColumn::Y) return i; } } else { // look to the right first for(int i=col+1; iplotDesignation() == AbstractColumn::Y) return i; } for(int i=col-1; i>=0; i--) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } } return -1; } /*! Sorts the given list of column. If 'leading' is a null pointer, each column is sorted separately. */ void Spreadsheet::sortColumns(Column *leading, QList cols, bool ascending) { if(cols.isEmpty()) return; // the normal QPair comparison does not work properly with descending sorting // thefore we use our own compare functions class CompareFunctions { public: static bool doubleLess(const QPair& a, const QPair& b) { return a.first < b.first; } static bool doubleGreater(const QPair& a, const QPair& b) { return a.first > b.first; } static bool QStringLess(const QPair& a, const QPair& b) { return a < b; } static bool QStringGreater(const QPair& a, const QPair& b) { return a > b; } static bool QDateTimeLess(const QPair& a, const QPair& b) { return a < b; } static bool QDateTimeGreater(const QPair& a, const QPair& b) { return a > b; } }; WAIT_CURSOR; beginMacro(i18n("%1: sort columns", name())); if(leading == 0) { // sort separately for (auto* col: cols) { switch (col->columnMode()) { case AbstractColumn::Numeric: { int rows = col->rowCount(); QList< QPair > map; for(int j=0; j(col->valueAt(j), j)); if(ascending) qStableSort(map.begin(), map.end(), CompareFunctions::doubleLess); else qStableSort(map.begin(), map.end(), CompareFunctions::doubleGreater); QListIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::Text: { int rows = col->rowCount(); QList< QPair > map; for(int j=0; j(col->textAt(j), j)); if(ascending) qStableSort(map.begin(), map.end(), CompareFunctions::QStringLess); else qStableSort(map.begin(), map.end(), CompareFunctions::QStringGreater); QListIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { int rows = col->rowCount(); QList< QPair > map; for(int j=0; j(col->dateTimeAt(j), j)); if(ascending) qStableSort(map.begin(), map.end(), CompareFunctions::QDateTimeLess); else qStableSort(map.begin(), map.end(), CompareFunctions::QDateTimeGreater); QListIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } } } } else { // sort with leading column switch (leading->columnMode()) { case AbstractColumn::Numeric: { QList< QPair > map; int rows = leading->rowCount(); for(int i=0; i(leading->valueAt(i), i)); if(ascending) qStableSort(map.begin(), map.end(), CompareFunctions::doubleLess); else qStableSort(map.begin(), map.end(), CompareFunctions::doubleGreater); QListIterator< QPair > it(map); foreach (Column *col, cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::Text: { QList< QPair > map; int rows = leading->rowCount(); for(int i=0; i(leading->textAt(i), i)); if(ascending) qStableSort(map.begin(), map.end(), CompareFunctions::QStringLess); else qStableSort(map.begin(), map.end(), CompareFunctions::QStringGreater); QListIterator< QPair > it(map); foreach (Column *col, cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { QList< QPair > map; int rows = leading->rowCount(); for(int i=0; i(leading->dateTimeAt(i), i)); if(ascending) qStableSort(map.begin(), map.end(), CompareFunctions::QDateTimeLess); else qStableSort(map.begin(), map.end(), CompareFunctions::QDateTimeGreater); QListIterator< QPair > it(map); foreach (Column *col, cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j=0; // put the values in the right order into temp_col while(it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } } } endMacro(); RESET_CURSOR; } // end of sortColumns() /*! Returns an icon to be used for decorating my views. */ QIcon Spreadsheet::icon() const { return QIcon::fromTheme("labplot-spreadsheet"); } /*! Returns the text displayed in the given cell. */ QString Spreadsheet::text(int row, int col) const { Column* c = column(col); if(!c) return QString(); return c->asStringColumn()->textAt(row); } /*! * This slot is, indirectly, called when a child of \c Spreadsheet (i.e. column) was selected in \c ProjectExplorer. * Emits the signal \c columnSelected that is handled in \c SpreadsheetView. */ void Spreadsheet::childSelected(const AbstractAspect* aspect) { const Column* column = qobject_cast(aspect); if (column) { int index = indexOfChild(column); emit columnSelected(index); } } /*! * This slot is, indirectly, called when a child of \c Spreadsheet (i.e. column) was deselected in \c ProjectExplorer. * Emits the signal \c columnDeselected that is handled in \c SpreadsheetView. */ void Spreadsheet::childDeselected(const AbstractAspect* aspect) { const Column* column = qobject_cast(aspect); if (column) { int index = indexOfChild(column); emit columnDeselected(index); } } /*! * Emits the signal to select or to deselect the column number \c index in the project explorer, * if \c selected=true or \c selected=false, respectively. * The signal is handled in \c AspectTreeModel and forwarded to the tree view in \c ProjectExplorer. * This function is called in \c SpreadsheetView upon selection changes. */ void Spreadsheet::setColumnSelectedInView(int index, bool selected) { if (selected){ emit childAspectSelectedInView(child(index)); //deselect the spreadsheet in the project explorer, if a child (column) was selected. //prevents unwanted multiple selection with spreadsheet (if it was selected before). emit childAspectDeselectedInView(this); } else { emit childAspectDeselectedInView(child(index)); } } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void Spreadsheet::save(QXmlStreamWriter* writer) const { writer->writeStartElement("spreadsheet"); writeBasicAttributes(writer); writeCommentElement(writer); //columns for (auto* col: children(IncludeHidden)) col->save(writer); writer->writeEndElement(); // "spreadsheet" } /*! Loads from XML. */ bool Spreadsheet::load(XmlStreamReader * reader) { if(reader->isStartElement() && reader->name() == "spreadsheet") { if (!readBasicAttributes(reader)) return false; // read child elements while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement()) break; if (reader->isStartElement()) { if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if(reader->name() == "column") { Column* column = new Column(""); if (!column->load(reader)) { delete column; setColumnCount(0); return false; } addChild(column); } else { // unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } } } else // no spreadsheet element reader->raiseError(i18n("no spreadsheet element found")); return !reader->hasError(); } //############################################################################## //######################## Data Import ####################################### //############################################################################## int Spreadsheet::prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode importMode, int actualRows, int actualCols, QStringList colNameList, QVector columnMode) { DEBUG("create() rows = " << actualRows << " cols = " << actualCols); int columnOffset = 0; setUndoAware(false); //make the available columns undo unaware before we resize and rename them below, //the same will be done for new columns in this->resize(). for (int i = 0; i < childCount(); i++) child(i)->setUndoAware(false); columnOffset = this->resize(importMode, colNameList, actualCols); // resize the spreadsheet if (importMode == AbstractFileFilter::Replace) { clear(); setRowCount(actualRows); } else { if (rowCount() < actualRows) setRowCount(actualRows); } dataContainer.resize(actualCols); for (int n = 0; n < actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) this->child(columnOffset+n)->setColumnMode(columnMode[n]); switch (columnMode[n]) { case AbstractColumn::Numeric: { QVector* vector = static_cast* >(this->child(columnOffset+n)->data()); vector->reserve(actualRows); vector->resize(actualRows); - dataContainer[n] = static_cast(vector); + dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { QVector* vector = static_cast*>(this->child(columnOffset+n)->data()); vector->reserve(actualRows); vector->resize(actualRows); - dataContainer[n] = static_cast(vector); + dataContainer[n] = static_cast(vector); break; } case AbstractColumn::DateTime: { QVector* vector = static_cast* >(this->child(columnOffset+n)->data()); vector->reserve(actualRows); vector->resize(actualRows); - dataContainer[n] = static_cast(vector); + dataContainer[n] = static_cast(vector); break; } //TODO case AbstractColumn::Month: case AbstractColumn::Day: break; } } // QDEBUG("dataPointers =" << dataPointers); return columnOffset; } /*! resize data source to cols columns returns column offset depending on import mode */ int Spreadsheet::resize(AbstractFileFilter::ImportMode mode, QStringList colNameList, int cols) { // name additional columns for (int k = colNameList.size(); k < cols; k++ ) colNameList.append( "Column " + QString::number(k+1) ); int columnOffset = 0; //indexes the "start column" in the spreadsheet. Starting from this column the data will be imported. Column* newColumn = 0; if (mode == AbstractFileFilter::Append) { columnOffset = childCount(); for (int n = 0; n < cols; n++ ) { newColumn = new Column(colNameList.at(n), AbstractColumn::Numeric); newColumn->setUndoAware(false); addChildFast(newColumn); } } else if (mode == AbstractFileFilter::Prepend) { Column* firstColumn = child(0); for (int n = 0; n < cols; n++ ) { newColumn = new Column(colNameList.at(n), AbstractColumn::Numeric); newColumn->setUndoAware(false); insertChildBeforeFast(newColumn, firstColumn); } } else if (mode == AbstractFileFilter::Replace) { //replace completely the previous content of the data source with the content to be imported. int columns = childCount(); if (columns > cols) { //there're more columns in the data source then required -> remove the superfluous columns for (int i = 0; i < columns-cols; i++) removeChild(child(0)); } else { //create additional columns if needed for (int i = columns; i < cols; i++) { newColumn = new Column(colNameList.at(i), AbstractColumn::Numeric); newColumn->setUndoAware(false); addChildFast(newColumn); } } //rename the columns that are already available and supress the dataChanged signal for them for (int i = 0; i < childCount(); i++) { if (mode == AbstractFileFilter::Replace) child(i)->setSuppressDataChangedSignal(true); child(i)->setName(colNameList.at(i)); } } return columnOffset; } void Spreadsheet::finalizeImport(int columnOffset, int startColumn, int endColumn, const QString& dateTimeFormat, AbstractFileFilter::ImportMode importMode) { // set the comments for each of the columns if datasource is a spreadsheet const int rows = rowCount(); for (int n = startColumn; n <= endColumn; n++) { Column* column = this->column(columnOffset + n - startColumn); QString comment; switch (column->columnMode()) { case AbstractColumn::Numeric: comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); break; case AbstractColumn::Text: comment = i18np("text data, %1 element", "text data, %1 elements", rows); break; case AbstractColumn::Month: comment = i18np("month data, %1 element", "month data, %1 elements", rows); break; case AbstractColumn::Day: comment = i18np("day data, %1 element", "day data, %1 elements", rows); break; case AbstractColumn::DateTime: comment = i18np("date and time data, %1 element", "date and time data, %1 elements", rows); // set same datetime format in column DateTime2StringFilter* filter = static_cast(column->outputFilter()); filter->setFormat(dateTimeFormat); } column->setComment(comment); if (importMode == AbstractFileFilter::Replace) { column->setSuppressDataChangedSignal(false); column->setChanged(); } } //make the spreadsheet and all its children undo aware again setUndoAware(true); for (int i=0; i(); i++) child(i)->setUndoAware(true); if (m_view) reinterpret_cast(m_view)->resizeHeader(); } diff --git a/src/kdefrontend/datasources/BinaryOptionsWidget.cpp b/src/kdefrontend/datasources/BinaryOptionsWidget.cpp index bf24b479f..cabf80ad1 100644 --- a/src/kdefrontend/datasources/BinaryOptionsWidget.cpp +++ b/src/kdefrontend/datasources/BinaryOptionsWidget.cpp @@ -1,93 +1,96 @@ /*************************************************************************** File : BinaryOptionsWidget.cpp Project : LabPlot Description : widget providing options for the import of binary data -------------------------------------------------------------------- Copyright : (C) 2009-2017 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009 by Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "BinaryOptionsWidget.h" #include "backend/datasources/filters/BinaryFilter.h" #include #include /*! \class BinaryOptionsWidget \brief Widget providing options for the import of binary data \ingroup kdefrontend */ BinaryOptionsWidget::BinaryOptionsWidget(QWidget* parent) : QWidget(parent) { ui.setupUi(parent); ui.cbDataType->addItems(BinaryFilter::dataTypes()); ui.cbByteOrder->addItems(BinaryFilter::byteOrders()); const QString textDataTypeShort = i18n("This option determines the data type that the imported data while converting to numbers."); ui.lDataType->setToolTip(textDataTypeShort); ui.lDataType->setWhatsThis(textDataTypeShort); ui.cbDataType->setToolTip(textDataTypeShort); ui.cbDataType->setWhatsThis(textDataTypeShort); const QString textByteOrderShort = i18n("This option determines the byte order of the imported data when converting to numbers."); const QString textByteOrder = textByteOrderShort + "

" + i18n( "" "" "" "
little endiantypical byte order (endianness) on Intel x86 processors.
big endiantypical byte order on Mainframes (IBM) and SPARC/PowerPC/Motorola processors.
"); ui.lByteOrder->setToolTip(textByteOrderShort); ui.lByteOrder->setWhatsThis(textByteOrder); ui.cbByteOrder->setToolTip(textByteOrderShort); ui.cbByteOrder->setWhatsThis(textByteOrder); } void BinaryOptionsWidget::applyFilterSettings(BinaryFilter* filter) const { Q_ASSERT(filter); filter->setVectors( ui.niVectors->value() ); filter->setDataType( (BinaryFilter::DataType)ui.cbDataType->currentIndex() ); + filter->setCreateIndexEnabled( ui.chbCreateIndex->isChecked() ); } void BinaryOptionsWidget::loadSettings() const { KConfigGroup conf(KSharedConfig::openConfig(), "Import"); ui.niVectors->setValue(conf.readEntry("Vectors", "2").toInt()); ui.cbDataType->setCurrentIndex(conf.readEntry("DataType", 0)); ui.cbByteOrder->setCurrentIndex(conf.readEntry("ByteOrder", 0)); ui.sbSkipStartBytes->setValue(conf.readEntry("SkipStartBytes", 0)); ui.sbSkipBytes->setValue(conf.readEntry("SkipBytes", 0)); + ui.chbCreateIndex->setChecked(conf.readEntry("CreateIndex", false)); } void BinaryOptionsWidget::saveSettings() { KConfigGroup conf(KSharedConfig::openConfig(), "Import"); conf.writeEntry("Vectors", ui.niVectors->value()); conf.writeEntry("ByteOrder", ui.cbByteOrder->currentIndex()); conf.writeEntry("DataType", ui.cbDataType->currentIndex()); conf.writeEntry("SkipStartBytes", ui.sbSkipStartBytes->value()); conf.writeEntry("SkipBytes", ui.sbSkipBytes->value()); + conf.writeEntry("CreateIndex", ui.chbCreateIndex->isChecked()); } diff --git a/src/kdefrontend/datasources/ImportFileWidget.cpp b/src/kdefrontend/datasources/ImportFileWidget.cpp index 2efe49577..63b789252 100644 --- a/src/kdefrontend/datasources/ImportFileWidget.cpp +++ b/src/kdefrontend/datasources/ImportFileWidget.cpp @@ -1,867 +1,866 @@ /*************************************************************************** File : ImportFileWidget.cpp Project : LabPlot Description : import file data widget -------------------------------------------------------------------- Copyright : (C) 2009-2017 Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2009-2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "ImportFileWidget.h" #include "FileInfoDialog.h" #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/filters/BinaryFilter.h" #include "backend/datasources/filters/HDFFilter.h" #include "backend/datasources/filters/NetCDFFilter.h" #include "backend/datasources/filters/ImageFilter.h" #include "backend/datasources/filters/FITSFilter.h" #include "AsciiOptionsWidget.h" #include "BinaryOptionsWidget.h" #include "HDFOptionsWidget.h" #include "ImageOptionsWidget.h" #include "NetCDFOptionsWidget.h" #include "FITSOptionsWidget.h" #include #include #include #include #include #include #include #include #include #include #include #include #include /*! \class ImportFileWidget \brief Widget for importing data from a file. \ingroup kdefrontend */ ImportFileWidget::ImportFileWidget(QWidget* parent, const QString& fileName) : QWidget(parent), m_fileName(fileName), m_fileDataSource(true) { ui.setupUi(this); KUrlCompletion *comp = new KUrlCompletion(); ui.kleFileName->setCompletionObject(comp); ui.cbFileType->addItems(FileDataSource::fileTypes()); QStringList filterItems; filterItems << i18n("Automatic") << i18n("Custom"); ui.cbFilter->addItems(filterItems); // file type specific option widgets QWidget* asciiw = new QWidget(); m_asciiOptionsWidget = std::unique_ptr(new AsciiOptionsWidget(asciiw)); ui.swOptions->insertWidget(FileDataSource::Ascii, asciiw); QWidget* binaryw = new QWidget(); m_binaryOptionsWidget = std::unique_ptr(new BinaryOptionsWidget(binaryw)); ui.swOptions->insertWidget(FileDataSource::Binary, binaryw); QWidget* imagew = new QWidget(); m_imageOptionsWidget = std::unique_ptr(new ImageOptionsWidget(imagew)); ui.swOptions->insertWidget(FileDataSource::Image, imagew); QWidget* hdfw = new QWidget(); m_hdfOptionsWidget = std::unique_ptr(new HDFOptionsWidget(hdfw, this)); ui.swOptions->insertWidget(FileDataSource::HDF, hdfw); QWidget* netcdfw = new QWidget(); m_netcdfOptionsWidget = std::unique_ptr(new NetCDFOptionsWidget(netcdfw, this)); ui.swOptions->insertWidget(FileDataSource::NETCDF, netcdfw); QWidget* fitsw = new QWidget(); m_fitsOptionsWidget = std::unique_ptr(new FITSOptionsWidget(fitsw, this)); ui.swOptions->insertWidget(FileDataSource::FITS, fitsw); // the table widget for preview m_twPreview = new QTableWidget(ui.tePreview); m_twPreview->verticalHeader()->hide(); m_twPreview->setEditTriggers(QTableWidget::NoEditTriggers); QHBoxLayout* layout = new QHBoxLayout; layout->addWidget(m_twPreview); ui.tePreview->setLayout(layout); m_twPreview->hide(); // default filter ui.swOptions->setCurrentIndex(FileDataSource::Ascii); #if !defined(HAVE_HDF5) || !defined(HAVE_NETCDF) || !defined(HAVE_FITS) const QStandardItemModel* model = qobject_cast(ui.cbFileType->model()); #endif #ifndef HAVE_HDF5 // disable HDF5 item QStandardItem* item = model->item(FileDataSource::HDF); item->setFlags(item->flags() & ~(Qt::ItemIsSelectable | Qt::ItemIsEnabled)); #endif #ifndef HAVE_NETCDF // disable NETCDF item QStandardItem* item2 = model->item(FileDataSource::NETCDF); item2->setFlags(item2->flags() & ~(Qt::ItemIsSelectable | Qt::ItemIsEnabled)); #endif #ifndef HAVE_FITS // disable FITS item QStandardItem* item3 = model->item(FileDataSource::FITS); item3->setFlags(item3->flags() & ~(Qt::ItemIsSelectable | Qt::ItemIsEnabled)); #endif ui.gbOptions->hide(); ui.gbUpdateOptions->hide(); ui.bOpen->setIcon( QIcon::fromTheme("document-open") ); ui.bFileInfo->setIcon( QIcon::fromTheme("help-about") ); ui.bManageFilters->setIcon( QIcon::fromTheme("configure") ); ui.bSaveFilter->setIcon( QIcon::fromTheme("document-save") ); ui.bRefreshPreview->setIcon( QIcon::fromTheme("view-refresh") ); connect( ui.kleFileName, SIGNAL(textChanged(QString)), SLOT(fileNameChanged(QString)) ); connect( ui.bOpen, SIGNAL(clicked()), this, SLOT (selectFile()) ); connect( ui.bFileInfo, SIGNAL(clicked()), this, SLOT (fileInfoDialog()) ); connect( ui.bSaveFilter, SIGNAL(clicked()), this, SLOT (saveFilter()) ); connect( ui.bManageFilters, SIGNAL(clicked()), this, SLOT (manageFilters()) ); connect( ui.cbFileType, SIGNAL(currentIndexChanged(int)), SLOT(fileTypeChanged(int)) ); connect( ui.cbUpdateOn, SIGNAL(currentIndexChanged(int)), this, SLOT(updateTypeChanged(int))); connect( ui.cbReadType, SIGNAL(currentIndexChanged(int)), this, SLOT(readingTypeChanged(int))); connect( ui.cbFilter, SIGNAL(activated(int)), SLOT(filterChanged(int)) ); connect( ui.bRefreshPreview, SIGNAL(clicked()), SLOT(refreshPreview()) ); connect( ui.cbSourceType, SIGNAL(currentIndexChanged(int)), this, SLOT(sourceTypeChanged(int))); //TODO: implement save/load of user-defined settings later and activate these buttons again ui.bSaveFilter->hide(); ui.bManageFilters->hide(); //defer the loading of settings a bit in order to show the dialog prior to blocking the GUI in refreshPreview() QTimer::singleShot( 100, this, SLOT(loadSettings()) ); } void ImportFileWidget::loadSettings() { //load last used settings KConfigGroup conf(KSharedConfig::openConfig(), "Import"); //settings for data type specific widgets m_asciiOptionsWidget->loadSettings(); m_binaryOptionsWidget->loadSettings(); m_imageOptionsWidget->loadSettings(); //general settings ui.cbFileType->setCurrentIndex(conf.readEntry("Type", 0)); ui.cbFilter->setCurrentIndex(conf.readEntry("Filter", 0)); filterChanged(ui.cbFilter->currentIndex()); // needed if filter is not changed if (m_fileName.isEmpty()) ui.kleFileName->setText(conf.readEntry("LastImportedFile", "")); else ui.kleFileName->setText(m_fileName); } ImportFileWidget::~ImportFileWidget() { // save current settings KConfigGroup conf(KSharedConfig::openConfig(), "Import"); // general settings conf.writeEntry("LastImportedFile", ui.kleFileName->text()); conf.writeEntry("Type", ui.cbFileType->currentIndex()); conf.writeEntry("Filter", ui.cbFilter->currentIndex()); // data type specific settings m_asciiOptionsWidget->saveSettings(); m_binaryOptionsWidget->saveSettings(); m_imageOptionsWidget->saveSettings(); } void ImportFileWidget::hideDataSource() { m_fileDataSource = false; ui.gbUpdateOptions->hide(); ui.chbWatchFile->hide(); ui.chbLinkFile->hide(); ui.cbBaudRate->hide(); ui.lBaudRate->hide(); ui.lHost->hide(); ui.leHost->hide(); ui.lPort->hide(); ui.lePort->hide(); ui.cbSerialPort->hide(); ui.lSerialPort->hide(); ui.lSourceType->hide(); ui.cbSourceType->hide(); ui.cbUpdateOn->hide(); ui.lUpdateOn->hide(); ui.sbUpdateFrequency->hide(); ui.lUpdateFrequency->hide(); } void ImportFileWidget::showAsciiHeaderOptions(bool b) { m_asciiOptionsWidget->showAsciiHeaderOptions(b); } void ImportFileWidget::showOptions(bool b) { ui.gbOptions->setVisible(b); if (m_fileDataSource) ui.gbUpdateOptions->setVisible(b); resize(layout()->minimumSize()); } QString ImportFileWidget::fileName() const { if (currentFileType() == FileDataSource::FITS) { QString extensionName = m_fitsOptionsWidget->currentExtensionName(); if (!extensionName.isEmpty()) return ui.kleFileName->text() + QLatin1String("[") + extensionName + QLatin1String("]"); } return ui.kleFileName->text(); } /*! saves the settings to the data source \c source. */ void ImportFileWidget::saveSettings(FileDataSource* source) const { //save the data source information FileDataSource::FileType fileType = static_cast(ui.cbFileType->currentIndex()); FileDataSource::UpdateType updateType = static_cast(ui.cbUpdateOn->currentIndex()); FileDataSource::SourceType sourceType = static_cast(ui.cbSourceType->currentIndex()); source->setComment( ui.kleFileName->text() ); source->setFileType(fileType); source->setFilter(this->currentFileFilter()); source->setUpdateType(updateType); source->setUpdateFrequency(ui.sbUpdateFrequency->value()); source->setSourceType(sourceType); source->setSampleRate(ui.sbSampleRate->value()); source->setKeepNvalues(ui.sbKeepValues->value()); if ((sourceType == FileDataSource::SourceType::FileOrPipe) || (sourceType == FileDataSource::SourceType::LocalSocket)) { source->setFileName( ui.kleFileName->text() ); source->setFileWatched( ui.chbWatchFile->isChecked() ); source->setFileLinked( ui.chbLinkFile->isChecked() ); } else if (sourceType == FileDataSource::SourceType::NetworkSocket) { source->setHost(ui.leHost->text()); source->setPort(ui.lePort->text().toInt()); } else if (sourceType == FileDataSource::SourceType::SerialPort) { source->setBaudRate(ui.cbBaudRate->currentText().toInt()); source->setSerialPort(ui.cbSerialPort->currentText()); } } /*! returns the currently used file type. */ FileDataSource::FileType ImportFileWidget::currentFileType() const { return static_cast(ui.cbFileType->currentIndex()); } /*! returns the currently used filter. */ AbstractFileFilter* ImportFileWidget::currentFileFilter() const { DEBUG("currentFileFilter()"); FileDataSource::FileType fileType = static_cast(ui.cbFileType->currentIndex()); switch (fileType) { case FileDataSource::Ascii: { //TODO std::unique_ptr filter(new AsciiFilter()); AsciiFilter* filter = new AsciiFilter(); if (ui.cbFilter->currentIndex() == 0) //"automatic" filter->setAutoModeEnabled(true); else if (ui.cbFilter->currentIndex() == 1) { //"custom" filter->setAutoModeEnabled(false); m_asciiOptionsWidget->applyFilterSettings(filter); } else filter->loadFilterSettings( ui.cbFilter->currentText() ); //save the data portion to import filter->setStartRow( ui.sbStartRow->value()); filter->setEndRow( ui.sbEndRow->value() ); filter->setStartColumn( ui.sbStartColumn->value()); filter->setEndColumn( ui.sbEndColumn->value()); return filter; } case FileDataSource::Binary: { BinaryFilter* filter = new BinaryFilter(); if ( ui.cbFilter->currentIndex() == 0 ) //"automatic" filter->setAutoModeEnabled(true); else if ( ui.cbFilter->currentIndex() == 1 ) { //"custom" filter->setAutoModeEnabled(false); m_binaryOptionsWidget->applyFilterSettings(filter); } else { //TODO: load filter settings // filter->setFilterName( ui.cbFilter->currentText() ); } filter->setStartRow( ui.sbStartRow->value() ); filter->setEndRow( ui.sbEndRow->value() ); return filter; } case FileDataSource::Image: { ImageFilter* filter = new ImageFilter(); filter->setImportFormat(m_imageOptionsWidget->currentFormat()); filter->setStartRow( ui.sbStartRow->value() ); filter->setEndRow( ui.sbEndRow->value() ); filter->setStartColumn( ui.sbStartColumn->value() ); filter->setEndColumn( ui.sbEndColumn->value() ); return filter; } case FileDataSource::HDF: { HDFFilter* filter = new HDFFilter(); QStringList names = selectedHDFNames(); if (!names.isEmpty()) filter->setCurrentDataSetName(names[0]); filter->setStartRow( ui.sbStartRow->value() ); filter->setEndRow( ui.sbEndRow->value() ); filter->setStartColumn( ui.sbStartColumn->value() ); filter->setEndColumn( ui.sbEndColumn->value() ); return filter; } case FileDataSource::NETCDF: { NetCDFFilter* filter = new NetCDFFilter(); if (!selectedNetCDFNames().isEmpty()) filter->setCurrentVarName(selectedNetCDFNames()[0]); filter->setStartRow( ui.sbStartRow->value() ); filter->setEndRow( ui.sbEndRow->value() ); filter->setStartColumn( ui.sbStartColumn->value() ); filter->setEndColumn( ui.sbEndColumn->value() ); return filter; } case FileDataSource::FITS: { FITSFilter* filter = new FITSFilter(); filter->setStartRow( ui.sbStartRow->value()); filter->setEndRow( ui.sbEndRow->value() ); filter->setStartColumn( ui.sbStartColumn->value()); filter->setEndColumn( ui.sbEndColumn->value()); return filter; } } return 0; } /*! opens a file dialog and lets the user select the file data source. */ void ImportFileWidget::selectFile() { KConfigGroup conf(KSharedConfig::openConfig(), "ImportFileWidget"); QString dir = conf.readEntry("LastDir", ""); QString path = QFileDialog::getOpenFileName(this, i18n("Select the File Data Source"), dir); if (path.isEmpty()) return; //cancel was clicked in the file-dialog int pos = path.lastIndexOf(QDir::separator()); if (pos != -1) { QString newDir = path.left(pos); if (newDir != dir) conf.writeEntry("LastDir", newDir); } ui.kleFileName->setText(path); //TODO: decide whether the selection of several files should be possible // QStringList filelist = QFileDialog::getOpenFileNames(this,i18n("Select one or more files to open")); // if (! filelist.isEmpty() ) // ui.kleFileName->setText(filelist.join(";")); } /************** SLOTS **************************************************************/ /*! called on file name changes. Determines the file format (ASCII, binary etc.), if the file exists, and activates the corresponding options. */ void ImportFileWidget::fileNameChanged(const QString& name) { QString fileName = name; #ifndef HAVE_WINDOWS // make relative path if ( !fileName.isEmpty() && fileName.left(1) != QDir::separator()) fileName = QDir::homePath() + QDir::separator() + fileName; #endif bool fileExists = QFile::exists(fileName); if (fileExists) ui.kleFileName->setStyleSheet(""); else ui.kleFileName->setStyleSheet("QLineEdit{background:red;}"); ui.gbOptions->setEnabled(fileExists); ui.bFileInfo->setEnabled(fileExists); ui.cbFileType->setEnabled(fileExists); ui.cbFilter->setEnabled(fileExists); ui.bManageFilters->setEnabled(fileExists); ui.chbWatchFile->setEnabled(fileExists); ui.chbLinkFile->setEnabled(fileExists); if (!fileExists) { //file doesn't exist -> delete the content preview that is still potentially //available from the previously selected file ui.tePreview->clear(); m_twPreview->clear(); m_hdfOptionsWidget->clear(); m_netcdfOptionsWidget->clear(); m_fitsOptionsWidget->clear(); emit fileNameChanged(); return; } QString fileInfo; #ifndef HAVE_WINDOWS //check, if we can guess the file type by content QProcess *proc = new QProcess(this); QStringList args; args << "-b" << ui.kleFileName->text(); proc->start("file", args); if (proc->waitForReadyRead(1000) == false) { QDEBUG("ERROR: reading file type of file" << fileName); return; } fileInfo = proc->readLine(); #endif QByteArray imageFormat = QImageReader::imageFormat(fileName); if (fileInfo.contains(QLatin1String("compressed data")) || fileInfo.contains(QLatin1String("ASCII")) || fileName.endsWith(QLatin1String("dat"), Qt::CaseInsensitive) || fileName.endsWith(QLatin1String("txt"), Qt::CaseInsensitive)) { //probably ascii data ui.cbFileType->setCurrentIndex(FileDataSource::Ascii); } else if (fileInfo.contains(QLatin1String("Hierarchical Data Format")) || fileName.endsWith(QLatin1String("h5"), Qt::CaseInsensitive) || fileName.endsWith(QLatin1String("hdf"), Qt::CaseInsensitive) || fileName.endsWith(QLatin1String("hdf5"), Qt::CaseInsensitive) ) { ui.cbFileType->setCurrentIndex(FileDataSource::HDF); // update HDF tree widget using current selected file m_hdfOptionsWidget->updateContent((HDFFilter*)this->currentFileFilter(), fileName); } else if (fileInfo.contains(QLatin1String("NetCDF Data Format")) || fileName.endsWith(QLatin1String("nc"), Qt::CaseInsensitive) || fileName.endsWith(QLatin1String("netcdf"), Qt::CaseInsensitive) || fileName.endsWith(QLatin1String("cdf"), Qt::CaseInsensitive)) { ui.cbFileType->setCurrentIndex(FileDataSource::NETCDF); // update NetCDF tree widget using current selected file m_netcdfOptionsWidget->updateContent((NetCDFFilter*)this->currentFileFilter(), fileName); } else if (fileInfo.contains(QLatin1String("FITS image data")) || fileName.endsWith(QLatin1String("fits"), Qt::CaseInsensitive) || fileName.endsWith(QLatin1String("fit"), Qt::CaseInsensitive) || fileName.endsWith(QLatin1String("fts"), Qt::CaseInsensitive)) { #ifdef HAVE_FITS ui.cbFileType->setCurrentIndex(FileDataSource::FITS); #endif // update FITS tree widget using current selected file m_fitsOptionsWidget->updateContent((FITSFilter*)this->currentFileFilter(), fileName); } else if (fileInfo.contains("image") || fileInfo.contains("bitmap") || !imageFormat.isEmpty()) ui.cbFileType->setCurrentIndex(FileDataSource::Image); else ui.cbFileType->setCurrentIndex(FileDataSource::Binary); refreshPreview(); emit fileNameChanged(); } /*! saves the current filter settings */ void ImportFileWidget::saveFilter() { bool ok; QString text = QInputDialog::getText(this, i18n("Save Filter Settings as"), i18n("Filter name:"), QLineEdit::Normal, i18n("new filter"), &ok); if (ok && !text.isEmpty()) { //TODO //AsciiFilter::saveFilter() } } /*! opens a dialog for managing all available predefined filters. */ void ImportFileWidget::manageFilters() { //TODO } /*! Depending on the selected file type, activates the corresponding options in the data portion tab and populates the combobox with the available pre-defined fllter settings for the selected type. */ void ImportFileWidget::fileTypeChanged(int fileType) { ui.swOptions->setCurrentIndex(fileType); //default ui.lFilter->show(); ui.cbFilter->show(); //if we switch from netCDF-format (only two tabs available), add the data preview-tab again if (ui.tabWidget->count() == 2) { ui.tabWidget->setTabText(0, i18n("Data format")); ui.tabWidget->insertTab(1, ui.tabDataPreview, i18n("Preview")); } ui.lPreviewLines->show(); ui.sbPreviewLines->show(); ui.lStartColumn->show(); ui.sbStartColumn->show(); ui.lEndColumn->show(); ui.sbEndColumn->show(); switch (fileType) { case FileDataSource::Ascii: break; case FileDataSource::Binary: ui.lStartColumn->hide(); ui.sbStartColumn->hide(); ui.lEndColumn->hide(); ui.sbEndColumn->hide(); break; case FileDataSource::HDF: case FileDataSource::NETCDF: ui.lFilter->hide(); ui.cbFilter->hide(); // hide global preview tab. we have our own ui.tabWidget->setTabText(0, i18n("Data format && preview")); ui.tabWidget->removeTab(1); ui.tabWidget->setCurrentIndex(0); break; case FileDataSource::Image: ui.lPreviewLines->hide(); ui.sbPreviewLines->hide(); ui.lFilter->hide(); ui.cbFilter->hide(); break; case FileDataSource::FITS: ui.lFilter->hide(); ui.cbFilter->hide(); ui.tabWidget->setTabText(0, i18n("Data format && preview")); ui.tabWidget->removeTab(1); ui.tabWidget->setCurrentIndex(0); break; default: DEBUG("unknown file type"); } m_hdfOptionsWidget->clear(); m_netcdfOptionsWidget->clear(); int lastUsedFilterIndex = ui.cbFilter->currentIndex(); ui.cbFilter->clear(); ui.cbFilter->addItem( i18n("Automatic") ); ui.cbFilter->addItem( i18n("Custom") ); //TODO: populate the combobox with the available pre-defined filter settings for the selected type ui.cbFilter->setCurrentIndex(lastUsedFilterIndex); filterChanged(lastUsedFilterIndex); refreshPreview(); } const QStringList ImportFileWidget::selectedHDFNames() const { return m_hdfOptionsWidget->selectedHDFNames(); } const QStringList ImportFileWidget::selectedNetCDFNames() const { return m_netcdfOptionsWidget->selectedNetCDFNames(); } const QStringList ImportFileWidget::selectedFITSExtensions() const { return m_fitsOptionsWidget->selectedFITSExtensions(); } /*! shows the dialog with the information about the file(s) to be imported. */ void ImportFileWidget::fileInfoDialog() { QStringList files = ui.kleFileName->text().split(';'); FileInfoDialog* dlg = new FileInfoDialog(this); dlg->setFiles(files); dlg->exec(); } /*! enables the options if the filter "custom" was chosen. Disables the options otherwise. */ void ImportFileWidget::filterChanged(int index) { // ignore filter for these formats if (ui.cbFileType->currentIndex() == FileDataSource::HDF || ui.cbFileType->currentIndex() == FileDataSource::NETCDF || ui.cbFileType->currentIndex() == FileDataSource::Image || ui.cbFileType->currentIndex() == FileDataSource::FITS) { ui.swOptions->setEnabled(true); return; } if (index == 0) { // "automatic" ui.swOptions->setEnabled(false); ui.bSaveFilter->setEnabled(false); } else if (index == 1) { //custom ui.swOptions->setEnabled(true); ui.bSaveFilter->setEnabled(true); } else { // predefined filter settings were selected. //load and show them in the GUI. //TODO } } void ImportFileWidget::refreshPreview() { DEBUG("refreshPreview()"); WAIT_CURSOR; QString fileName = ui.kleFileName->text(); #ifndef HAVE_WINDOWS if (fileName.left(1) != QDir::separator()) fileName = QDir::homePath() + QDir::separator() + fileName; #endif QVector importedStrings; FileDataSource::FileType fileType = (FileDataSource::FileType)ui.cbFileType->currentIndex(); // generic table widget if (fileType == FileDataSource::Ascii || fileType == FileDataSource::Binary) m_twPreview->show(); else m_twPreview->hide(); int lines = ui.sbPreviewLines->value(); bool ok = true; QTableWidget *tmpTableWidget = nullptr; QStringList vectorNameList; QVector columnModes; switch (fileType) { case FileDataSource::Ascii: { ui.tePreview->clear(); AsciiFilter *filter = (AsciiFilter *)this->currentFileFilter(); importedStrings = filter->preview(fileName, lines); tmpTableWidget = m_twPreview; vectorNameList = filter->vectorNames(); columnModes = filter->columnModes(); break; } case FileDataSource::Binary: { ui.tePreview->clear(); - BinaryFilter *filter = (BinaryFilter *)this->currentFileFilter(); importedStrings = filter->readDataFromFile(fileName, nullptr, AbstractFileFilter::Replace, lines); tmpTableWidget = m_twPreview; break; } case FileDataSource::Image: { ui.tePreview->clear(); QImage image(fileName); QTextCursor cursor = ui.tePreview->textCursor(); cursor.insertImage(image); RESET_CURSOR; return; } case FileDataSource::HDF: { HDFFilter *filter = (HDFFilter *)this->currentFileFilter(); lines = m_hdfOptionsWidget->lines(); importedStrings = filter->readCurrentDataSet(fileName, NULL, ok, AbstractFileFilter::Replace, lines); tmpTableWidget = m_hdfOptionsWidget->previewWidget(); break; } case FileDataSource::NETCDF: { NetCDFFilter *filter = (NetCDFFilter *)this->currentFileFilter(); lines = m_netcdfOptionsWidget->lines(); importedStrings = filter->readCurrentVar(fileName, NULL, AbstractFileFilter::Replace, lines); tmpTableWidget = m_netcdfOptionsWidget->previewWidget(); break; } case FileDataSource::FITS: { FITSFilter* filter = (FITSFilter*)this->currentFileFilter(); lines = m_fitsOptionsWidget->lines(); QString extensionName = m_fitsOptionsWidget->extensionName(&ok); if (!extensionName.isEmpty()) fileName = extensionName; bool readFitsTableToMatrix; importedStrings = filter->readChdu(fileName, &readFitsTableToMatrix, lines); emit checkedFitsTableToMatrix(readFitsTableToMatrix); tmpTableWidget = m_fitsOptionsWidget->previewWidget(); break; } } // fill the table widget tmpTableWidget->setRowCount(0); tmpTableWidget->setColumnCount(0); if( !importedStrings.isEmpty() ) { QDEBUG("importedStrings =" << importedStrings); if (!ok) { // show imported strings as error message tmpTableWidget->setRowCount(1); tmpTableWidget->setColumnCount(1); QTableWidgetItem* item = new QTableWidgetItem(); item->setText(importedStrings[0][0]); tmpTableWidget->setItem(0, 0, item); } else { //TODO: maxrows not used const int rows = qMax(importedStrings.size(), 1); const int maxColumns = 300; tmpTableWidget->setRowCount(rows); for (int i = 0; i < rows; i++) { QDEBUG(importedStrings[i]); int cols = importedStrings[i].size() > maxColumns ? maxColumns : importedStrings[i].size(); // new if (cols > tmpTableWidget->columnCount()) tmpTableWidget->setColumnCount(cols); for (int j = 0; j < cols; j++) { QTableWidgetItem* item = new QTableWidgetItem(importedStrings[i][j]); tmpTableWidget->setItem(i, j, item); } } // set header if columnMode available for (int i = 0; i < qMin(tmpTableWidget->columnCount(), columnModes.size()); i++) { QString columnName = QString::number(i+1); if (i < vectorNameList.size()) columnName = vectorNameList[i]; auto* item = new QTableWidgetItem(columnName + QLatin1String(" {") + ENUM_TO_STRING(AbstractColumn, ColumnMode, columnModes[i]) + QLatin1String("}")); item->setTextAlignment(Qt::AlignLeft); item->setIcon(AbstractColumn::iconForMode(columnModes[i])); tmpTableWidget->setHorizontalHeaderItem(i, item); } } tmpTableWidget->horizontalHeader()->resizeSections(QHeaderView::ResizeToContents); } RESET_CURSOR; } void ImportFileWidget::updateTypeChanged(int idx) { FileDataSource::UpdateType type = static_cast(idx); if (type == FileDataSource::UpdateType::TimeInterval) { ui.lUpdateFrequency->show(); ui.sbUpdateFrequency->show(); } else if (type == FileDataSource::UpdateType::NewData) { ui.lUpdateFrequency->hide(); ui.sbUpdateFrequency->hide(); } } void ImportFileWidget::readingTypeChanged(int idx) { FileDataSource::ReadingType type = static_cast(idx); if (type == FileDataSource::ReadingType::TillEnd) { ui.lSampleRate->hide(); ui.sbSampleRate->hide(); } else { ui.lSampleRate->show(); ui.sbSampleRate->show(); } } void ImportFileWidget::sourceTypeChanged(int idx) { FileDataSource::SourceType type = static_cast(idx); if ((type == FileDataSource::SourceType::FileOrPipe) || (type == FileDataSource::SourceType::LocalSocket)) { ui.lFileName->show(); ui.kleFileName->show(); ui.bFileInfo->show(); ui.bOpen->show(); ui.cbBaudRate->hide(); ui.lBaudRate->hide(); ui.lHost->hide(); ui.leHost->hide(); ui.lPort->hide(); ui.lePort->hide(); ui.cbSerialPort->hide(); ui.lSerialPort->hide(); } else if (type == FileDataSource::SourceType::NetworkSocket) { ui.lHost->show(); ui.leHost->show(); ui.lePort->show(); ui.lPort->show(); ui.lBaudRate->hide(); ui.cbBaudRate->hide(); ui.lSerialPort->hide(); ui.cbSerialPort->hide(); ui.lFileName->hide(); ui.kleFileName->hide(); ui.bFileInfo->hide(); ui.bOpen->hide(); } else if (type == FileDataSource::SourceType::SerialPort) { ui.lBaudRate->show(); ui.cbBaudRate->show(); ui.lSerialPort->show(); ui.cbSerialPort->show(); ui.lHost->hide(); ui.leHost->hide(); ui.lePort->hide(); ui.lPort->hide(); ui.lFileName->hide(); ui.kleFileName->hide(); ui.bFileInfo->hide(); ui.bOpen->hide(); } } void ImportFileWidget::initializeAndFillPortsAndBaudRates() { for (int i = 2; i < ui.swOptions->count(); ++i) ui.swOptions->removeWidget(ui.swOptions->widget(i)); const int size = ui.cbFileType->count(); for (int i = 2; i < size; ++i) ui.cbFileType->removeItem(2); ui.cbBaudRate->hide(); ui.lBaudRate->hide(); ui.lHost->hide(); ui.leHost->hide(); ui.lPort->hide(); ui.lePort->hide(); ui.cbSerialPort->hide(); ui.lSerialPort->hide(); ui.cbBaudRate->addItems(FileDataSource::supportedBaudRates()); ui.cbSerialPort->addItems(FileDataSource::availablePorts()); ui.tabDataPortion->hide(); } diff --git a/src/kdefrontend/ui/datasources/binaryoptionswidget.ui b/src/kdefrontend/ui/datasources/binaryoptionswidget.ui index 498fbc802..7e6267f63 100644 --- a/src/kdefrontend/ui/datasources/binaryoptionswidget.ui +++ b/src/kdefrontend/ui/datasources/binaryoptionswidget.ui @@ -1,121 +1,128 @@ BinaryOptionsWidget 0 0 - 289 - 202 + 295 + 240 - - - - Skip bytes at start of file - - - 2147483647 - - - - - - - - - - Skip start bytes: - - - - - + + - Vectors: + Skip Bytes: The number of vectors in the file 1 2 Qt::Horizontal 80 20 Data type: + + + + Skip bytes at start of file + + + 2147483647 + + + + + + + Skip bytes after each value + + + 2147483647 + + + + + + + Vectors: + + + + + + + + + + Skip start bytes: + + + Byte order: - + Qt::Vertical 20 74 - - + + - Skip Bytes: - - - - - - - Skip bytes after each value - - - 2147483647 + Create index column