diff --git a/src/backend/datasources/filters/HDF5Filter.cpp b/src/backend/datasources/filters/HDF5Filter.cpp index 337888926..6a74b9a96 100644 --- a/src/backend/datasources/filters/HDF5Filter.cpp +++ b/src/backend/datasources/filters/HDF5Filter.cpp @@ -1,1926 +1,1931 @@ /*************************************************************************** File : HDF5Filter.cpp Project : LabPlot Description : HDF5 I/O-filter -------------------------------------------------------------------- Copyright : (C) 2015-2018 by Stefan Gerlach (stefan.gerlach@uni.kn) Copyright : (C) 2017 Alexander Semke (alexander.semke@web.de) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ /* TODO: * Feature: implement missing data types and ranks * Performance: only fill dataPointer or dataStrings (not both) */ #include "backend/datasources/filters/HDF5Filter.h" #include "backend/datasources/filters/HDF5FilterPrivate.h" #include "backend/datasources/LiveDataSource.h" #include "backend/core/column/Column.h" #include #include #include #include /*! \class HDF5Filter \brief Manages the import/export of data from/to a HDF5 file. \ingroup datasources */ HDF5Filter::HDF5Filter():AbstractFileFilter(HDF5), d(new HDF5FilterPrivate(this)) {} HDF5Filter::~HDF5Filter() = default; /*! parses the content of the file \c fileName. */ void HDF5Filter::parse(const QString & fileName, QTreeWidgetItem* rootItem) { d->parse(fileName, rootItem); } /*! reads the content of the data set \c dataSet from file \c fileName. */ QVector HDF5Filter::readCurrentDataSet(const QString& fileName, AbstractDataSource* dataSource, bool &ok, AbstractFileFilter::ImportMode importMode, int lines) { return d->readCurrentDataSet(fileName, dataSource, ok, importMode, lines); } /*! reads the content of the file \c fileName to the data source \c dataSource. */ void HDF5Filter::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { d->readDataFromFile(fileName, dataSource, mode); } /*! writes the content of the data source \c dataSource to the file \c fileName. */ void HDF5Filter::write(const QString& fileName, AbstractDataSource* dataSource) { d->write(fileName, dataSource); } /////////////////////////////////////////////////////////////////////// /*! loads the predefined filter settings for \c filterName */ void HDF5Filter::loadFilterSettings(const QString& filterName) { Q_UNUSED(filterName); } /*! saves the current settings as a new filter with the name \c filterName */ void HDF5Filter::saveFilterSettings(const QString& filterName) const { Q_UNUSED(filterName); } /////////////////////////////////////////////////////////////////////// void HDF5Filter::setCurrentDataSetName(const QString& ds) { d->currentDataSetName = ds; } const QString HDF5Filter::currentDataSetName() const { return d->currentDataSetName; } void HDF5Filter::setStartRow(const int s) { d->startRow = s; } int HDF5Filter::startRow() const { return d->startRow; } void HDF5Filter::setEndRow(const int e) { d->endRow = e; } int HDF5Filter::endRow() const { return d->endRow; } void HDF5Filter::setStartColumn(const int c) { d->startColumn = c; } int HDF5Filter::startColumn() const { return d->startColumn; } void HDF5Filter::setEndColumn(const int c) { d->endColumn = c; } int HDF5Filter::endColumn() const { return d->endColumn; } QString HDF5Filter::fileInfoString(const QString& fileName) { DEBUG("HDF5Filter::fileInfoString()"); QString info; #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); DEBUG("fileName = " << bafileName.data()); // check file type first htri_t isHdf5 = H5Fis_hdf5(bafileName.data()); if (isHdf5 == 0) { DEBUG(bafileName.data() << " is not a HDF5 file! isHdf5 = " << isHdf5 << " Giving up."); return i18n("Not a HDF5 file"); } if (isHdf5 < 0) { DEBUG("H5Fis_hdf5() failed on " << bafileName.data() << "! Giving up."); return i18n("Failed checking file"); } // open file hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); HDF5FilterPrivate::handleError((int)file, "H5Fopen", fileName); if (file < 0) { DEBUG("Opening file " << bafileName.data() << " failed! Giving up."); return i18n("Failed opening HDF5 file"); } hsize_t size; herr_t status = H5Fget_filesize(file, &size); if (status >= 0) { info += i18n("File size: %1 bytes", QString::number(size)); info += QLatin1String("
"); } hssize_t freesize = H5Fget_freespace(file); info += i18n("Free space: %1 bytes", QString::number(freesize)); info += QLatin1String("
"); info += QLatin1String("
"); ssize_t objectCount; objectCount = H5Fget_obj_count(file, H5F_OBJ_FILE); info += i18n("Number of files: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_DATASET); info += i18n("Number of data sets: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_GROUP); info += i18n("Number of groups: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_DATATYPE); info += i18n("Number of named datatypes: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_ATTR); info += i18n("Number of attributes: %1", QString::number(objectCount)); info += QLatin1String("
"); objectCount = H5Fget_obj_count(file, H5F_OBJ_ALL); info += i18n("Number of all objects: %1", QString::number(objectCount)); info += QLatin1String("
"); #ifdef HAVE_AT_LEAST_HDF5_1_10_0 // using H5Fget_info2 struct (see H5Fpublic.h) H5F_info2_t file_info; status = H5Fget_info2(file, &file_info); if (status >= 0) { info += QLatin1String("
"); info += i18n("Version of superblock: %1", QString::number(file_info.super.version)); info += QLatin1String("
"); info += i18n("Size of superblock: %1 bytes", QString::number(file_info.super.super_size)); info += QLatin1String("
"); info += i18n("Size of superblock extension: %1 bytes", QString::number(file_info.super.super_ext_size)); info += QLatin1String("
"); info += i18n("Version of free-space manager: %1", QString::number(file_info.free.version)); info += QLatin1String("
"); info += i18n("Size of free-space manager metadata: %1 bytes", QString::number(file_info.free.meta_size)); info += QLatin1String("
"); info += i18n("Total size of free space: %1 bytes", QString::number(file_info.free.tot_space)); info += QLatin1String("
"); info += i18n("Version of shared object header: %1", QString::number(file_info.sohm.version)); info += QLatin1String("
"); info += i18n("Size of shared object header: %1 bytes", QString::number(file_info.sohm.hdr_size)); info += QLatin1String("
"); info += i18n("Size of all shared object header indexes: %1 bytes", QString::number(file_info.sohm.msgs_info.index_size)); info += QLatin1String("
"); info += i18n("Size of the heap: %1 bytes", QString::number(file_info.sohm.msgs_info.heap_size)); info += QLatin1String("
"); } #else // using H5Fget_info1 struct (named H5F_info_t in HDF5 1.8) H5F_info_t file_info; status = H5Fget_info(file, &file_info); if (status >= 0) { info += i18n("Size of superblock extension: %1 bytes", QString::number(file_info.super_ext_size)); info += QLatin1String("
"); info += i18n("Size of shared object header: %1 bytes", QString::number(file_info.sohm.hdr_size)); info += QLatin1String("
"); info += i18n("Size of all shared object header indexes: %1 bytes", QString::number(file_info.sohm.msgs_info.index_size)); info += QLatin1String("
"); info += i18n("Size of the heap: %1 bytes", QString::number(file_info.sohm.msgs_info.heap_size)); info += QLatin1String("
"); } #endif // cache information //see https://support.hdfgroup.org/HDF5/doc/RM/RM_H5F.html info += QLatin1String("
"); H5AC_cache_config_t config; config.version = H5AC__CURR_CACHE_CONFIG_VERSION; status = H5Fget_mdc_config(file, &config); if (status >= 0) { info += i18n("Cache config version: %1", QString::number(config.version)); info += QLatin1String("
"); info += i18n("Adaptive cache resize report function enabled: %1", config.rpt_fcn_enabled ? i18n("Yes") : i18n("No")); info += QLatin1String("
"); info += i18n("Cache initial maximum size: %1 bytes", QString::number(config.initial_size)); info += QLatin1String("
"); info += i18n("Adaptive cache maximum size: %1 bytes", QString::number(config.max_size)); info += QLatin1String("
"); info += i18n("Adaptive cache minimum size: %1 bytes", QString::number(config.min_size)); info += QLatin1String("
"); //TODO: more settings } double hit_rate; status = H5Fget_mdc_hit_rate(file, &hit_rate); info += i18n("Metadata cache hit rate: %1", QString::number(hit_rate)); info += QLatin1String("
"); //TODO: herr_t H5Fget_mdc_image_info(hid_t file_id, haddr_t *image_addr, hsize_t *image_len) size_t max_size, min_clean_size, cur_size; int cur_num_entries; status = H5Fget_mdc_size(file, &max_size, &min_clean_size, &cur_size, &cur_num_entries); if (status >= 0) { info += i18n("Current cache maximum size: %1 bytes", QString::number(max_size)); info += QLatin1String("
"); info += i18n("Current cache minimum clean size: %1 bytes", QString::number(min_clean_size)); info += QLatin1String("
"); info += i18n("Current cache size: %1 bytes", QString::number(cur_size)); info += QLatin1String("
"); info += i18n("Current number of entries in the cache: %1", QString::number(cur_num_entries)); info += QLatin1String("
"); } //TODO: 1.10 herr_t H5Fget_metadata_read_retry_info( hid_t file_id, H5F_retry_info_t *info ) /* TODO: not available hbool_t atomicMode; status = H5Fget_mpi_atomicity(file, &atomicMode); if (status >= 0) { info += i18n("MPI file access atomic mode: %1", atomicMode ? i18n("Yes") : i18n("No")); info += QLatin1String("
"); }*/ #ifdef HAVE_AT_LEAST_HDF5_1_10_0 hbool_t is_enabled, is_currently_logging; status = H5Fget_mdc_logging_status(file, &is_enabled, &is_currently_logging); if (status >= 0) { info += i18n("Logging enabled: %1", is_enabled ? i18n("Yes") : i18n("No")); info += QLatin1String("
"); info += i18n("Events are currently logged: %1", is_currently_logging ? i18n("Yes") : i18n("No")); info += QLatin1String("
"); } #endif #ifdef HAVE_AT_LEAST_HDF5_1_10_1 unsigned int accesses[2], hits[2], misses[2], evictions[2], bypasses[2]; status = H5Fget_page_buffering_stats(file, accesses, hits, misses, evictions, bypasses); if (status >= 0) { info += i18n("Metadata/raw data page buffer accesses: %1 %2", QString::number(accesses[0]), QString::number(accesses[1])); info += QLatin1String("
"); info += i18n("Metadata/raw data page buffer hits: %1 %2", QString::number(hits[0]), QString::number(hits[1])); info += QLatin1String("
"); info += i18n("Metadata/raw data page buffer misses: %1 %2", QString::number(misses[0]), QString::number(misses[1])); info += QLatin1String("
"); info += i18n("Metadata/raw data page buffer evictions: %1 %2", QString::number(evictions[0]), QString::number(evictions[1])); info += QLatin1String("
"); info += i18n("Metadata/raw data accesses bypassing page buffer: %1 %2", QString::number(bypasses[0]), QString::number(bypasses[1])); info += QLatin1String("
"); } else { info += i18n("Page buffer disabled"); info += QLatin1String("
"); DEBUG("H5Fget_page_buffering_stats() status = " << status); } #endif #else Q_UNUSED(fileName); #endif return info; } /*! * Get file content in DDL (Data Description Language) format * uses "h5dump" */ QString HDF5Filter::fileDDLString(const QString& fileName) { DEBUG("HDF5Filter::fileDDLString()"); QString DDLString; #ifdef Q_OS_LINUX auto* proc = new QProcess(); QStringList args; args << "-H" << fileName; proc->start( "h5dump", args); if (proc->waitForReadyRead(1000) == false) DDLString += i18n("Reading from file %1 failed.", fileName); else { DDLString += proc->readAll(); DDLString.replace('\n', "
\n"); DDLString.replace("\t","    "); //DEBUG(" DDL string: " << DDLString.toStdString()); } #else //TODO: h5dump on Win, Mac Q_UNUSED(fileName) #endif return DDLString; } //##################################################################### //################### Private implementation ########################## //##################################################################### HDF5FilterPrivate::HDF5FilterPrivate(HDF5Filter* owner) : q(owner) { #ifdef HAVE_HDF5 m_status = 0; #endif } #ifdef HAVE_HDF5 void HDF5FilterPrivate::handleError(int err, const QString& function, const QString& arg) { #ifdef NDEBUG Q_UNUSED(err) Q_UNUSED(function) Q_UNUSED(arg) #else if (err < 0) { DEBUG("ERROR " << err << ": " << function.toStdString() << "() - " << arg.toStdString()); } #endif } QString HDF5FilterPrivate::translateHDF5Order(H5T_order_t o) { QString order; switch (o) { case H5T_ORDER_LE: order = "LE"; break; case H5T_ORDER_BE: order = "BE"; break; case H5T_ORDER_VAX: order = "VAX"; break; case H5T_ORDER_MIXED: order = "MIXED"; break; case H5T_ORDER_NONE: order = "NONE"; break; case H5T_ORDER_ERROR: order = "ERROR"; break; } return order; } QString HDF5FilterPrivate::translateHDF5Type(hid_t t) { QString type; if (H5Tequal(t, H5T_STD_I8LE) || H5Tequal(t, H5T_STD_I8BE)) type = "CHAR"; else if (H5Tequal(t, H5T_STD_U8LE) || H5Tequal(t, H5T_STD_U8BE)) type = "UCHAR"; else if (H5Tequal(t, H5T_STD_I16LE) || H5Tequal(t, H5T_STD_I16BE)) type = "SHORT"; else if (H5Tequal(t, H5T_STD_U16LE) || H5Tequal(t, H5T_STD_U16BE)) type = "USHORT"; else if (H5Tequal(t, H5T_STD_I32LE) || H5Tequal(t, H5T_STD_I32BE)) type = "INT"; else if (H5Tequal(t, H5T_STD_U32LE) || H5Tequal(t, H5T_STD_U32BE)) type = "UINT"; else if (H5Tequal(t, H5T_NATIVE_LONG)) type = "LONG"; else if (H5Tequal(t, H5T_NATIVE_ULONG)) type = "ULONG"; else if (H5Tequal(t, H5T_STD_I64LE) || H5Tequal(t, H5T_STD_I64BE)) type = "LLONG"; else if (H5Tequal(t, H5T_STD_U64LE) || H5Tequal(t, H5T_STD_U64BE)) type = "ULLONG"; else if (H5Tequal(t, H5T_IEEE_F32LE) || H5Tequal(t, H5T_IEEE_F32BE)) type = "FLOAT"; else if (H5Tequal(t, H5T_IEEE_F64LE) || H5Tequal(t, H5T_IEEE_F64BE)) type = "DOUBLE"; else if (H5Tequal(t, H5T_NATIVE_LDOUBLE)) type = "LDOUBLE"; else type = "UNKNOWN"; return type; } QString HDF5FilterPrivate::translateHDF5Class(H5T_class_t c) { QString dclass; switch (c) { case H5T_INTEGER: dclass = "INTEGER"; break; case H5T_FLOAT: dclass = "FLOAT"; break; case H5T_STRING: dclass = "STRING"; break; case H5T_BITFIELD: dclass = "BITFIELD"; break; case H5T_OPAQUE: dclass = "OPAQUE"; break; case H5T_COMPOUND: dclass = "COMPOUND"; break; case H5T_ARRAY: dclass = "ARRAY"; break; case H5T_ENUM: dclass = "ENUM"; break; case H5T_REFERENCE: dclass = "REFERENCE"; break; case H5T_VLEN: dclass = "VLEN"; break; case H5T_TIME: dclass = "TIME"; break; case H5T_NCLASSES: dclass = "NCLASSES"; break; case H5T_NO_CLASS: dclass = "NOCLASS"; break; } return dclass; } QStringList HDF5FilterPrivate::readHDF5Compound(hid_t tid) { size_t typeSize = H5Tget_size(tid); QString line; line += QLatin1String("COMPOUND(") + QString::number(typeSize) + QLatin1String(") : ("); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); for (int i = 0; i < members; ++i) { H5T_class_t mclass = H5Tget_member_class(tid, i); handleError((int)mclass, "H5Tget_member_class"); hid_t mtype = H5Tget_member_type(tid, i); handleError((int)mtype, "H5Tget_member_type"); size_t size = H5Tget_size(mtype); handleError((int)size, "H5Tget_size"); QString typeString = translateHDF5Class(mclass); if (mclass == H5T_INTEGER || mclass == H5T_FLOAT) typeString = translateHDF5Type(mtype); line += H5Tget_member_name(tid, i) + QLatin1String("[") + typeString + QLatin1String("(") + QString::number(size) + QLatin1String(")]"); if (i == members-1) line += QLatin1String(")"); else line += QLatin1String(","); m_status = H5Tclose(mtype); handleError(m_status, "H5Tclose"); } QStringList dataString; dataString << line; return dataString; } template QStringList HDF5FilterPrivate::readHDF5Data1D(hid_t dataset, hid_t type, int rows, int lines, void* dataContainer) { DEBUG("readHDF5Data1D() rows = " << rows << ", lines = " << lines); QStringList dataString; // we read all rows of data T* data = new T[rows]; m_status = H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Dread"); DEBUG(" startRow = " << startRow << ", endRow = " << endRow); DEBUG(" dataContainer = " << dataContainer); for (int i = startRow - 1; i < qMin(endRow, lines + startRow - 1); ++i) { if (dataContainer) // read to data source static_cast*>(dataContainer)->operator[](i-startRow+1) = data[i]; else // for preview dataString << QString::number(static_cast(data[i])); } delete[] data; return dataString; } QStringList HDF5FilterPrivate::readHDF5CompoundData1D(hid_t dataset, hid_t tid, int rows, int lines, QVector& dataContainer) { DEBUG("HDF5FilterPrivate::readHDF5CompoundData1D()"); DEBUG(" dataContainer size = " << dataContainer.size()); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); DEBUG(" # members = " << members); QStringList dataString; if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) dataString << QLatin1String("("); dataContainer.resize(members); // avoid "index out of range" for preview } for (int m = 0; m < members; ++m) { hid_t mtype = H5Tget_member_type(tid, m); handleError((int)mtype, "H5Tget_member_type"); size_t msize = H5Tget_size(mtype); handleError((int)msize, "H5Tget_size"); hid_t ctype = H5Tcreate(H5T_COMPOUND, msize); handleError((int)ctype, "H5Tcreate"); m_status = H5Tinsert(ctype, H5Tget_member_name(tid, m), 0, mtype); handleError(m_status, "H5Tinsert"); QStringList mdataString; if (H5Tequal(mtype, H5T_STD_I8LE) || H5Tequal(mtype, H5T_STD_I8BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 2: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 4: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 8: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; } } else if (H5Tequal(mtype, H5T_STD_U8LE) || H5Tequal(mtype, H5T_STD_U8BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 2: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 4: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; case 8: mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); break; } } else if (H5Tequal(mtype, H5T_STD_I16LE) || H5Tequal(mtype, H5T_STD_I16BE) || H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U16LE) || H5Tequal(mtype, H5T_STD_U16BE) || H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_I32LE) || H5Tequal(mtype, H5T_STD_I32BE) || H5Tequal(mtype, H5T_NATIVE_INT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U32LE) || H5Tequal(mtype, H5T_STD_U32BE) || H5Tequal(mtype, H5T_NATIVE_UINT)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_LONG)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_ULONG)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_I64LE) || H5Tequal(mtype, H5T_STD_I64BE) || H5Tequal(mtype, H5T_NATIVE_LLONG)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_STD_U64LE) || H5Tequal(mtype, H5T_STD_U64BE) || H5Tequal(mtype, H5T_NATIVE_ULLONG)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_IEEE_F32LE) || H5Tequal(mtype, H5T_IEEE_F32BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_IEEE_F64LE) || H5Tequal(mtype, H5T_IEEE_F64BE)) mdataString = readHDF5Data1D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, lines, dataContainer[m]); else if (H5Tequal(mtype, H5T_NATIVE_LDOUBLE)) mdataString = readHDF5Data1D(dataset, ctype, rows, lines, dataContainer[m]); else { if (dataContainer[m]) { for (int i = startRow-1; i < qMin(endRow, lines + startRow - 1); ++i) static_cast*>(dataContainer[m])->operator[](i - startRow + 1) = 0; } else { for (int i = 0; i < qMin(rows, lines); ++i) mdataString << QLatin1String("_"); } H5T_class_t mclass = H5Tget_member_class(tid, m); handleError((int)mclass, "H5Tget_member_class"); DEBUG("unsupported type of class " << translateHDF5Class(mclass).toStdString()); } if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) { dataString[i] += mdataString[i]; if (m < members - 1) dataString[i] += QLatin1String(","); } } H5Tclose(ctype); } if (!dataContainer[0]) { for (int i = 0; i < qMin(rows, lines); ++i) dataString[i] += QLatin1String(")"); } return dataString; } template QVector HDF5FilterPrivate::readHDF5Data2D(hid_t dataset, hid_t type, int rows, int cols, int lines, QVector& dataPointer) { DEBUG("readHDF5Data2D() rows = " << rows << ", cols =" << cols << ", lines =" << lines); QVector dataStrings; T** data = (T**) malloc(rows*sizeof(T*)); data[0] = (T*) malloc(cols*rows*sizeof(T)); for (int i = 1; i < rows; ++i) data[i] = data[0] + i*cols; m_status = H5Dread(dataset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data[0][0]); handleError(m_status,"H5Dread"); for (int i = 0; i < qMin(rows, lines); ++i) { QStringList line; line.reserve(cols); for (int j = 0; j < cols; ++j) { if (dataPointer[0]) static_cast*>(dataPointer[j-startColumn+1])->operator[](i-startRow+1) = data[i][j]; else line << QString::number(static_cast(data[i][j])); } dataStrings << line; } free(data[0]); free(data); QDEBUG(dataStrings); return dataStrings; } QVector HDF5FilterPrivate::readHDF5CompoundData2D(hid_t dataset, hid_t tid, int rows, int cols, int lines) { DEBUG("readHDF5CompoundData2D() rows =" << rows << "cols =" << cols << "lines =" << lines); int members = H5Tget_nmembers(tid); handleError(members, "H5Tget_nmembers"); DEBUG(" # members =" << members); QVector dataStrings; for (int i = 0; i < qMin(rows, lines); ++i) { QStringList lineStrings; for (int j = 0; j < cols; ++j) lineStrings << QLatin1String("("); dataStrings << lineStrings; } //QStringList* data = new QStringList[members]; for (int m = 0; m < members; ++m) { hid_t mtype = H5Tget_member_type(tid, m); handleError((int)mtype, "H5Tget_member_type"); size_t msize = H5Tget_size(mtype); handleError((int)msize, "H5Tget_size"); hid_t ctype = H5Tcreate(H5T_COMPOUND, msize); handleError((int)ctype, "H5Tcreate"); m_status = H5Tinsert(ctype, H5Tget_member_name(tid, m), 0, mtype); handleError(m_status, "H5Tinsert"); // dummy container for all data columns // initially contains one pointer set to NULL QVector dummy(1, nullptr); QVector mdataStrings; if (H5Tequal(mtype, H5T_STD_I8LE) || H5Tequal(mtype, H5T_STD_I8BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 2: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 4: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 8: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; } } else if (H5Tequal(mtype, H5T_STD_U8LE) || H5Tequal(mtype, H5T_STD_U8BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 2: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 4: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; case 8: mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); break; } } else if (H5Tequal(mtype, H5T_STD_I16LE) || H5Tequal(mtype, H5T_STD_I16BE)|| H5Tequal(mtype, H5T_NATIVE_SHORT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U16LE) || H5Tequal(mtype, H5T_STD_U16BE) || H5Tequal(mtype, H5T_NATIVE_USHORT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_I32LE) || H5Tequal(mtype, H5T_STD_I32BE) || H5Tequal(mtype, H5T_NATIVE_INT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U32LE) || H5Tequal(mtype, H5T_STD_U32BE) || H5Tequal(mtype, H5T_NATIVE_UINT)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_LONG)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_ULONG)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_I64LE) || H5Tequal(mtype, H5T_STD_I64BE) || H5Tequal(mtype, H5T_NATIVE_LLONG)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_STD_U64LE) || H5Tequal(mtype, H5T_STD_U64BE) || H5Tequal(mtype, H5T_NATIVE_ULLONG)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_IEEE_F32LE) || H5Tequal(mtype, H5T_IEEE_F32BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_IEEE_F64LE) || H5Tequal(mtype, H5T_IEEE_F64BE)) mdataStrings = readHDF5Data2D(dataset, H5Tget_native_type(ctype, H5T_DIR_DEFAULT), rows, cols, lines, dummy); else if (H5Tequal(mtype, H5T_NATIVE_LDOUBLE)) mdataStrings = readHDF5Data2D(dataset, ctype, rows, cols, lines, dummy); else { for (int i = 0; i < qMin(rows, lines); ++i) { QStringList lineString; for (int j = 0; j < cols; ++j) lineString << QLatin1String("_"); mdataStrings << lineString; } #ifndef NDEBUG H5T_class_t mclass = H5Tget_member_class(tid, m); #endif DEBUG("unsupported class " << translateHDF5Class(mclass).toStdString()); } m_status = H5Tclose(ctype); handleError(m_status, "H5Tclose"); for (int i = 0; i < qMin(rows, lines); i++) { for (int j = 0; j < cols; j++) { dataStrings[i][j] += mdataStrings[i][j]; if (m < members-1) dataStrings[i][j] += QLatin1String(","); } } } for (int i = 0; i < qMin(rows, lines); ++i) { for (int j = 0; j < cols; ++j) dataStrings[i][j] += QLatin1String(")"); } QDEBUG("dataStrings =" << dataStrings); return dataStrings; } QStringList HDF5FilterPrivate::readHDF5Attr(hid_t aid) { QStringList attr; char name[MAXNAMELENGTH]; m_status = H5Aget_name(aid, MAXNAMELENGTH, name); handleError(m_status, "H5Aget_name"); attr << QString(name); // DEBUG(" name =" << QString(name)); hid_t aspace = H5Aget_space(aid); // the dimensions of the attribute data handleError((int)aspace, "H5Aget_space"); hid_t atype = H5Aget_type(aid); handleError((int)atype, "H5Aget_type"); hid_t aclass = H5Tget_class(atype); handleError((int)aclass, "H5Aget_class"); if (aclass == H5T_STRING) { char buf[MAXSTRINGLENGTH]; // buffer to read attr value hid_t amem = H5Tget_native_type(atype, H5T_DIR_ASCEND); handleError((int)amem, "H5Tget_native_type"); m_status = H5Aread(aid, amem, buf); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString(buf); m_status = H5Tclose(amem); handleError(m_status, "H5Tclose"); } else if (aclass == H5T_INTEGER) { if (H5Tequal(atype, H5T_STD_I8LE)) { qint8 value; m_status = H5Aread(aid, H5T_STD_I8LE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I8BE)) { qint8 value; m_status = H5Aread(aid, H5T_STD_I8BE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: { qint8 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 2: { qint16 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 4: { qint32 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 8: { qint64 value; m_status = H5Aread(aid, H5T_NATIVE_CHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } default: DEBUG("unknown size " << sizeof(H5T_NATIVE_CHAR) << " of H5T_NATIVE_CHAR"); return QStringList(QString()); } } else if (H5Tequal(atype, H5T_STD_U8LE)) { uint8_t value; m_status = H5Aread(aid, H5T_STD_U8LE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U8BE)) { uint8_t value; m_status = H5Aread(aid, H5T_STD_U8BE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: { uint8_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 2: { uint16_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 4: { uint32_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } case 8: { uint64_t value; m_status = H5Aread(aid, H5T_NATIVE_UCHAR, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); break; } default: DEBUG("unknown size " << sizeof(H5T_NATIVE_UCHAR) << " of H5T_NATIVE_UCHAR"); return QStringList(QString()); } } else if (H5Tequal(atype, H5T_STD_I16LE) || H5Tequal(atype, H5T_STD_I16BE) || H5Tequal(atype, H5T_NATIVE_SHORT)) { short value; m_status = H5Aread(aid, H5T_NATIVE_SHORT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U16LE) || H5Tequal(atype, H5T_STD_U16BE) || H5Tequal(atype, H5T_NATIVE_USHORT)) { unsigned short value; m_status = H5Aread(aid, H5T_NATIVE_USHORT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I32LE) || H5Tequal(atype, H5T_STD_I32BE) || H5Tequal(atype, H5T_NATIVE_INT)) { int value; m_status = H5Aread(aid, H5T_NATIVE_INT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U32LE) || H5Tequal(atype, H5T_STD_U32BE) || H5Tequal(atype, H5T_NATIVE_UINT)) { unsigned int value; m_status = H5Aread(aid, H5T_NATIVE_UINT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_LONG)) { long value; m_status = H5Aread(aid, H5T_NATIVE_LONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_ULONG)) { unsigned long value; m_status = H5Aread(aid, H5T_NATIVE_ULONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_I64LE) || H5Tequal(atype, H5T_STD_I64BE) || H5Tequal(atype, H5T_NATIVE_LLONG)) { long long value; m_status = H5Aread(aid, H5T_NATIVE_LLONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_STD_U64LE) || H5Tequal(atype, H5T_STD_U64BE) || H5Tequal(atype, H5T_NATIVE_ULLONG)) { unsigned long long value; m_status = H5Aread(aid, H5T_NATIVE_ULLONG, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else attr<<" (unknown integer)"; } else if (aclass == H5T_FLOAT) { if (H5Tequal(atype, H5T_IEEE_F32LE) || H5Tequal(atype, H5T_IEEE_F32BE)) { float value; m_status = H5Aread(aid, H5T_NATIVE_FLOAT, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_IEEE_F64LE) || H5Tequal(atype, H5T_IEEE_F64BE)) { double value; m_status = H5Aread(aid, H5T_NATIVE_DOUBLE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number(value); } else if (H5Tequal(atype, H5T_NATIVE_LDOUBLE)) { long double value; m_status = H5Aread(aid, H5T_NATIVE_LDOUBLE, &value); handleError(m_status, "H5Aread"); attr << QLatin1String("=") << QString::number((double)value); } else attr<<" (unknown float)"; } m_status = H5Tclose(atype); handleError(m_status, "H5Tclose"); m_status = H5Sclose(aspace); handleError(m_status, "H5Sclose"); return attr; } QStringList HDF5FilterPrivate::scanHDF5Attrs(hid_t oid) { QStringList attrList; int numAttr = H5Aget_num_attrs(oid); handleError(numAttr, "H5Aget_num_attrs"); DEBUG("number of attr = " << numAttr); for (int i = 0; i < numAttr; ++i) { hid_t aid = H5Aopen_idx(oid, i); handleError((int)aid, "H5Aopen_idx"); attrList << readHDF5Attr(aid); if (i < numAttr-1) attrList << QLatin1String(", "); m_status = H5Aclose(aid); handleError(m_status, "H5Aclose"); } return attrList; } QStringList HDF5FilterPrivate::readHDF5DataType(hid_t tid) { H5T_class_t typeClass = H5Tget_class(tid); handleError((int)typeClass, "H5Tget_class"); QStringList typeProps; QString typeString = translateHDF5Class(typeClass); if (typeClass == H5T_INTEGER || typeClass == H5T_FLOAT) typeString = translateHDF5Type(tid); typeProps<setIcon(0, QIcon::fromTheme("accessories-calculator")); dataTypeItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(dataTypeItem); } void HDF5FilterPrivate::scanHDF5DataSet(hid_t did, char *dataSetName, QTreeWidgetItem* parentItem) { QString attr = scanHDF5Attrs(did).join(QString()); char link[MAXNAMELENGTH]; m_status = H5Iget_name(did, link, MAXNAMELENGTH); handleError(m_status, "H5Iget_name"); QStringList dataSetProps; hsize_t size = H5Dget_storage_size(did); handleError((int)size, "H5Dget_storage_size"); hid_t datatype = H5Dget_type(did); handleError((int)datatype, "H5Dget_type"); size_t typeSize = H5Tget_size(datatype); handleError((int)typeSize, "H5Dget_size"); dataSetProps << readHDF5DataType(datatype); hid_t dataspace = H5Dget_space(did); int rank = H5Sget_simple_extent_ndims(dataspace); handleError(rank, "H5Sget_simple_extent_ndims"); unsigned int rows = 1, cols = 1, regs = 1; if (rank == 1) { hsize_t dims_out[1]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, nullptr); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else if (rank == 2) { hsize_t dims_out[2]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, nullptr); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; cols = dims_out[1]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String("x") << QString::number(cols) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else if (rank == 3) { hsize_t dims_out[3]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, nullptr); handleError(m_status, "H5Sget_simple_extent_dims"); rows = dims_out[0]; cols = dims_out[1]; regs = dims_out[2]; dataSetProps << QLatin1String(", ") << QString::number(rows) << QLatin1String("x") << QString::number(cols) << QLatin1String("x") << QString::number(regs) << QLatin1String(" (") << QString::number(size/typeSize) << QLatin1String(")"); } else dataSetProps << QLatin1String(", ") << i18n("rank %1 not supported yet", rank); hid_t pid = H5Dget_create_plist(did); handleError((int)pid, "H5Dget_create_plist"); dataSetProps << ", " << readHDF5PropertyList(pid).join(QString()); QTreeWidgetItem* dataSetItem = new QTreeWidgetItem(QStringList()<setIcon(0, QIcon::fromTheme("x-office-spreadsheet")); for (int i = 0; i < dataSetItem->columnCount(); ++i) { if (rows > 0 && cols > 0 && regs > 0) { dataSetItem->setBackground(i, QColor(192,255,192)); dataSetItem->setForeground(i, Qt::black); dataSetItem->setFlags(Qt::ItemIsEnabled | Qt::ItemIsSelectable); } else dataSetItem->setFlags(Qt::NoItemFlags); } parentItem->addChild(dataSetItem); } void HDF5FilterPrivate::scanHDF5Link(hid_t gid, char *linkName, QTreeWidgetItem* parentItem) { char target[MAXNAMELENGTH]; m_status = H5Gget_linkval(gid, linkName, MAXNAMELENGTH, target) ; handleError(m_status, "H5Gget_linkval"); QTreeWidgetItem* linkItem = new QTreeWidgetItem(QStringList() << QString(linkName) << i18n("symbolic link") << i18n("link to %1", QFile::decodeName(target))); linkItem->setIcon(0, QIcon::fromTheme("emblem-symbolic-link")); linkItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(linkItem); } void HDF5FilterPrivate::scanHDF5Group(hid_t gid, char *groupName, QTreeWidgetItem* parentItem) { DEBUG("HDF5FilterPrivate::scanHDF5Group()"); //check for hard link H5G_stat_t statbuf; m_status = H5Gget_objinfo(gid, ".", true, &statbuf); handleError(m_status, "H5Gget_objinfo"); if (statbuf.nlink > 1) { if (m_multiLinkList.contains(statbuf.objno[0])) { QTreeWidgetItem* objectItem = new QTreeWidgetItem(QStringList()<setIcon(0, QIcon::fromTheme("link")); objectItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(objectItem); return; } else { m_multiLinkList.append(statbuf.objno[0]); DEBUG(" group multiple links: "<setIcon(0, QIcon::fromTheme("folder")); groupItem->setFlags(Qt::ItemIsEnabled); parentItem->addChild(groupItem); hsize_t numObj; m_status = H5Gget_num_objs(gid, &numObj); handleError(m_status, "H5Gget_num_objs"); for (unsigned int i = 0; i < numObj; ++i) { char memberName[MAXNAMELENGTH]; m_status = H5Gget_objname_by_idx(gid, (hsize_t)i, memberName, (size_t)MAXNAMELENGTH ); handleError(m_status, "H5Gget_objname_by_idx"); int otype = H5Gget_objtype_by_idx(gid, (size_t)i ); handleError(otype, "H5Gget_objtype_by_idx"); switch (otype) { case H5G_LINK: { scanHDF5Link(gid, memberName, groupItem); break; } case H5G_GROUP: { hid_t grpid = H5Gopen(gid, memberName, H5P_DEFAULT); handleError((int)grpid, "H5Gopen"); scanHDF5Group(grpid, memberName, groupItem); m_status = H5Gclose(grpid); handleError(m_status, "H5Gclose"); break; } case H5G_DATASET: { hid_t dsid = H5Dopen(gid, memberName, H5P_DEFAULT); handleError((int)dsid, "H5Dopen"); scanHDF5DataSet(dsid, memberName, groupItem); m_status = H5Dclose(dsid); handleError(m_status, "H5Dclose"); break; } case H5G_TYPE: { hid_t tid = H5Topen(gid, memberName, H5P_DEFAULT); handleError((int)tid, "H5Topen"); scanHDF5DataType(tid, memberName, groupItem); m_status = H5Tclose(tid); handleError(m_status, "H5Tclose"); break; } default: QTreeWidgetItem* objectItem = new QTreeWidgetItem(QStringList() << QString(memberName) << i18n("unknown")); objectItem->setFlags(Qt::ItemIsEnabled); groupItem->addChild(objectItem); break; } } } #endif /*! parses the content of the file \c fileName and fill the tree using rootItem. */ void HDF5FilterPrivate::parse(const QString& fileName, QTreeWidgetItem* rootItem) { DEBUG("HDF5FilterPrivate::parse()"); #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); DEBUG("fileName = " << bafileName.data()); // check file type first htri_t isHdf5 = H5Fis_hdf5(bafileName.data()); if (isHdf5 == 0) { DEBUG(bafileName.data() << " is not a HDF5 file! Giving up."); return; } if (isHdf5 < 0) { DEBUG("H5Fis_hdf5() failed on " << bafileName.data() << "! Giving up."); return; } // open file hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); handleError((int)file, "H5Fopen", fileName); if (file < 0) { DEBUG("Opening file " << bafileName.data() << " failed! Giving up."); return; } char rootName[] = "/"; hid_t group = H5Gopen(file, rootName, H5P_DEFAULT); handleError((int)group, "H5Gopen", rootName); // multiLinkList.clear(); crashes scanHDF5Group(group, rootName, rootItem); m_status = H5Gclose(group); handleError(m_status, "H5Gclose", QString()); m_status = H5Fclose(file); handleError(m_status, "H5Fclose", QString()); #else DEBUG("HDF5 not available"); Q_UNUSED(fileName) Q_UNUSED(rootItem) #endif } /*! reads the content of the date set in the file \c fileName to a string (for preview) or to the data source. */ QVector HDF5FilterPrivate::readCurrentDataSet(const QString& fileName, AbstractDataSource* dataSource, bool &ok, AbstractFileFilter::ImportMode mode, int lines) { DEBUG("HDF5Filter::readCurrentDataSet()"); QVector dataStrings; if (currentDataSetName.isEmpty()) { //return QString("No data set selected").replace(' ',QChar::Nbsp); ok = false; return dataStrings << (QStringList() << i18n("No data set selected")); } DEBUG(" current data set = " << currentDataSetName.toStdString()); #ifdef HAVE_HDF5 QByteArray bafileName = fileName.toLatin1(); hid_t file = H5Fopen(bafileName.data(), H5F_ACC_RDONLY, H5P_DEFAULT); handleError((int)file, "H5Fopen", fileName); QByteArray badataSet = currentDataSetName.toLatin1(); hid_t dataset = H5Dopen2(file, badataSet.data(), H5P_DEFAULT); handleError((int)file, "H5Dopen2", currentDataSetName); // Get datatype and dataspace hid_t dtype = H5Dget_type(dataset); handleError((int)dtype, "H5Dget_type"); H5T_class_t dclass = H5Tget_class(dtype); handleError((int)dclass, "H5Dget_class"); size_t typeSize = H5Tget_size(dtype); handleError((int)(typeSize-1), "H5Dget_size"); hid_t dataspace = H5Dget_space(dataset); handleError((int)dataspace, "H5Dget_space"); int rank = H5Sget_simple_extent_ndims(dataspace); handleError(rank, "H5Dget_simple_extent_ndims"); DEBUG(" rank = " << rank); int columnOffset = 0; // offset to import data int actualRows = 0, actualCols = 0; // rows and cols to read // dataContainer is used to store the data read from the dataSource // it contains the pointers of all columns // initially there is one pointer set to nullptr // check for dataContainer[0] != nullptr to decide if dataSource can be used QVector dataContainer(1, nullptr); // rank= 0: single value, 1: vector, 2: matrix, 3: 3D data, ... switch (rank) { case 0: { // single value actualCols = 1; switch (dclass) { case H5T_STRING: { char* data = (char *) malloc(typeSize * sizeof(char)); hid_t memtype = H5Tcopy(H5T_C_S1); handleError((int)memtype, "H5Tcopy"); m_status = H5Tset_size(memtype, typeSize); handleError(m_status, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Tread"); dataStrings << (QStringList() << data); free(data); break; } case H5T_INTEGER: case H5T_FLOAT: case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_COMPOUND: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataStrings << (QStringList() << i18n("rank 0 not implemented yet for type %1", translateHDF5Class(dclass))); QDEBUG(dataStrings); } default: break; } break; } case 1: { // vector hsize_t size, maxSize; m_status = H5Sget_simple_extent_dims(dataspace, &size, &maxSize); handleError(m_status, "H5Sget_simple_extent_dims"); int rows = size; if (endRow == -1) endRow = rows; if (lines == -1) lines = endRow; actualRows = endRow - startRow + 1; actualCols = 1; #ifndef NDEBUG H5T_order_t order = H5Tget_order(dtype); handleError((int)order, "H5Sget_order"); #endif QDEBUG(translateHDF5Class(dclass) << '(' << typeSize << ')' << translateHDF5Order(order) << ", rows:" << rows << " max:" << maxSize); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); - //TODO: use given names? - QStringList vectorNames; + // use current data set name (without path) for column name + QStringList vectorNames = {currentDataSetName.mid(currentDataSetName.lastIndexOf("/") + 1)}; + QDEBUG(" vector names = " << vectorNames) if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); QStringList dataString; // data saved in a list switch (dclass) { case H5T_STRING: { DEBUG("rank 1 H5T_STRING"); hid_t memtype = H5Tcopy(H5T_C_S1); handleError((int)memtype, "H5Tcopy"); char** data = (char **) malloc(rows * sizeof (char *)); if (H5Tis_variable_str(dtype)) { m_status = H5Tset_size(memtype, H5T_VARIABLE); handleError((int)memtype, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); handleError(m_status, "H5Dread"); } else { data[0] = (char *) malloc(rows * typeSize * sizeof (char)); for (int i = 1; i < rows; ++i) data[i] = data[0] + i * typeSize; m_status = H5Tset_size(memtype, typeSize); handleError((int)memtype, "H5Tset_size"); m_status = H5Dread(dataset, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, data[0]); handleError(m_status, "H5Dread"); } for (int i = startRow-1; i < qMin(endRow, lines + startRow - 1); ++i) dataString << data[i]; free(data); break; } case H5T_INTEGER: { if (H5Tequal(dtype, H5T_STD_I8LE)) dataString = readHDF5Data1D(dataset, H5T_STD_I8LE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I8BE)) dataString = readHDF5Data1D(dataset, H5T_STD_I8BE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 2: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 4: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; case 8: dataString = readHDF5Data1D(dataset, H5T_NATIVE_CHAR, rows, lines, dataContainer[0]); break; } } else if (H5Tequal(dtype, H5T_STD_U8LE)) dataString = readHDF5Data1D(dataset, H5T_STD_U8LE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U8BE)) dataString = readHDF5Data1D(dataset, H5T_STD_U8BE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 2: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 4: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; case 8: dataString = readHDF5Data1D(dataset, H5T_NATIVE_UCHAR, rows, lines, dataContainer[0]); break; } } else if (H5Tequal(dtype, H5T_STD_I16LE) || H5Tequal(dtype, H5T_STD_I16BE) || H5Tequal(dtype, H5T_NATIVE_SHORT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_SHORT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U16LE) || H5Tequal(dtype, H5T_STD_U16BE) || H5Tequal(dtype, H5T_NATIVE_USHORT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_USHORT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I32LE) || H5Tequal(dtype, H5T_STD_I32BE) || H5Tequal(dtype, H5T_NATIVE_INT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_INT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U32LE) || H5Tequal(dtype, H5T_STD_U32BE) || H5Tequal(dtype, H5T_NATIVE_UINT)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_UINT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_LONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_ULONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_ULONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_I64LE) || H5Tequal(dtype, H5T_STD_I64BE) || H5Tequal(dtype, H5T_NATIVE_LLONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LLONG, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_STD_U64LE) || H5Tequal(dtype, H5T_STD_U64BE) || H5Tequal(dtype, H5T_NATIVE_ULLONG)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_ULLONG, rows, lines, dataContainer[0]); else { ok = false; dataString = (QStringList() << i18n("unsupported integer type for rank 1")); QDEBUG(dataString); } break; } case H5T_FLOAT: { if (H5Tequal(dtype, H5T_IEEE_F32LE) || H5Tequal(dtype, H5T_IEEE_F32BE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_FLOAT, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_IEEE_F64LE) || H5Tequal(dtype, H5T_IEEE_F64BE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_DOUBLE, rows, lines, dataContainer[0]); else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) dataString = readHDF5Data1D(dataset, H5T_NATIVE_LDOUBLE, rows, lines, dataContainer[0]); else { ok = false; dataString = (QStringList() << i18n("unsupported float type for rank 1")); QDEBUG(dataString); } break; } case H5T_COMPOUND: { int members = H5Tget_nmembers(dtype); handleError(members, "H5Tget_nmembers"); if (dataSource) { // re-create data pointer dataContainer.clear(); dataSource->prepareImport(dataContainer, mode, actualRows, members, vectorNames, columnModes); } else dataStrings << readHDF5Compound(dtype); dataString = readHDF5CompoundData1D(dataset, dtype, rows, lines, dataContainer); break; } case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataString = (QStringList() << i18n("rank 1 not implemented yet for type %1", translateHDF5Class(dclass))); QDEBUG(dataString); } default: break; } if (!dataSource) { // preview QDEBUG("dataString =" << dataString); DEBUG(" data string size = " << dataString.size()); DEBUG(" rows = " << rows << ", lines = " << lines << ", actual rows = " << actualRows); for (int i = 0; i < qMin(actualRows, lines); ++i) dataStrings << (QStringList() << dataString[i]); } break; } case 2: { // matrix hsize_t dims_out[2]; m_status = H5Sget_simple_extent_dims(dataspace, dims_out, nullptr); handleError(m_status, "H5Sget_simple_extent_dims"); int rows = dims_out[0]; int cols = dims_out[1]; if (endRow == -1) endRow = rows; if (lines == -1) lines = endRow; if (endColumn == -1) endColumn = cols; actualRows = endRow-startRow+1; actualCols = endColumn-startColumn+1; #ifndef NDEBUG H5T_order_t order = H5Tget_order(dtype); handleError((int)order, "H5Tget_order"); #endif QDEBUG(translateHDF5Class(dclass) << '(' << typeSize << ')' << translateHDF5Order(order) << "," << rows << "x" << cols); DEBUG("startRow/endRow" << startRow << endRow); DEBUG("startColumn/endColumn" << startColumn << endColumn); DEBUG("actual rows/cols" << actualRows << actualCols); DEBUG("lines" << lines); //TODO: support other modes QVector columnModes; columnModes.resize(actualCols); - //TODO: use given names? + // use current data set name (without path) append by "_" and column number for column names QStringList vectorNames; + QString colName = currentDataSetName.mid(currentDataSetName.lastIndexOf("/") + 1); + for (int i = 0; i < actualCols; i++) + vectorNames << colName + QLatin1String("_") + QString::number(i + 1); + QDEBUG(" vector names = " << vectorNames) if (dataSource) columnOffset = dataSource->prepareImport(dataContainer, mode, actualRows, actualCols, vectorNames, columnModes); // read data switch (dclass) { case H5T_INTEGER: { if (H5Tequal(dtype, H5T_STD_I8LE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_I8LE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I8BE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_I8BE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_CHAR)) { switch (sizeof(H5T_NATIVE_CHAR)) { case 1: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 2: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 4: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; case 8: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_CHAR, rows, cols, lines, dataContainer); break; } } else if (H5Tequal(dtype, H5T_STD_U8LE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_U8LE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U8BE)) dataStrings << readHDF5Data2D(dataset, H5T_STD_U8BE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_UCHAR)) { switch (sizeof(H5T_NATIVE_UCHAR)) { case 1: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 2: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 4: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; case 8: dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UCHAR, rows, cols, lines, dataContainer); break; } } else if (H5Tequal(dtype, H5T_STD_I16LE) || H5Tequal(dtype, H5T_STD_I16BE) || H5Tequal(dtype, H5T_NATIVE_SHORT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_SHORT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U16LE) || H5Tequal(dtype, H5T_STD_U16BE) || H5Tequal(dtype, H5T_NATIVE_USHORT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_USHORT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I32LE) || H5Tequal(dtype, H5T_STD_I32BE) || H5Tequal(dtype, H5T_NATIVE_INT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_INT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U32LE) || H5Tequal(dtype, H5T_STD_U32BE) || H5Tequal(dtype, H5T_NATIVE_UINT)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_UINT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_LONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_ULONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_ULONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_I64LE) || H5Tequal(dtype, H5T_STD_I64BE) || H5Tequal(dtype, H5T_NATIVE_LLONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LLONG, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_STD_U64LE) || H5Tequal(dtype, H5T_STD_U64BE) || H5Tequal(dtype, H5T_NATIVE_ULLONG)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_ULLONG, rows, cols, lines, dataContainer); else { ok = false; dataStrings << (QStringList() << i18n("unsupported integer type for rank 2")); QDEBUG(dataStrings); } break; } case H5T_FLOAT: { if (H5Tequal(dtype, H5T_IEEE_F32LE) || H5Tequal(dtype, H5T_IEEE_F32BE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_FLOAT, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_IEEE_F64LE) || H5Tequal(dtype, H5T_IEEE_F64BE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_DOUBLE, rows, cols, lines, dataContainer); else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) dataStrings << readHDF5Data2D(dataset, H5T_NATIVE_LDOUBLE, rows, cols, lines, dataContainer); else { ok = false; dataStrings << (QStringList() << i18n("unsupported float type for rank 2")); QDEBUG(dataStrings); } break; } case H5T_COMPOUND: { dataStrings << readHDF5Compound(dtype); QDEBUG(dataStrings); dataStrings << readHDF5CompoundData2D(dataset,dtype,rows,cols,lines); break; } case H5T_STRING: { // TODO: implement this ok = false; dataStrings << (QStringList() << i18n("rank 2 not implemented yet for type %1, size = %2", translateHDF5Class(dclass), typeSize)); QDEBUG(dataStrings); break; } case H5T_TIME: case H5T_BITFIELD: case H5T_OPAQUE: case H5T_REFERENCE: case H5T_ENUM: case H5T_VLEN: case H5T_ARRAY: case H5T_NO_CLASS: case H5T_NCLASSES: { ok = false; dataStrings << (QStringList() << i18n("rank 2 not implemented yet for type %1", translateHDF5Class(dclass))); QDEBUG(dataStrings); } default: break; } break; } default: { // 3D or more data ok = false; dataStrings << (QStringList() << i18n("rank %1 not implemented yet for type %2", rank, translateHDF5Class(dclass))); QDEBUG(dataStrings); } } m_status = H5Sclose(dataspace); handleError(m_status, "H5Sclose"); m_status = H5Tclose(dtype); handleError(m_status, "H5Tclose"); m_status = H5Dclose(dataset); handleError(m_status, "H5Dclose"); m_status = H5Fclose(file); handleError(m_status, "H5Fclose"); if (!dataSource) return dataStrings; dataSource->finalizeImport(columnOffset, 1, actualCols, -1, QString(), mode); #else Q_UNUSED(fileName) Q_UNUSED(dataSource) Q_UNUSED(mode) Q_UNUSED(lines) #endif return dataStrings; } /*! reads the content of the file \c fileName to the data source \c dataSource. Uses the settings defined in the data source. */ void HDF5FilterPrivate::readDataFromFile(const QString& fileName, AbstractDataSource* dataSource, AbstractFileFilter::ImportMode mode) { DEBUG("HDF5Filter::readDataFromFile()"); if (currentDataSetName.isEmpty()) { - DEBUG("No data set selected"); + DEBUG("WARNING: No data set selected"); return; } bool ok = true; readCurrentDataSet(fileName, dataSource, ok, mode); } /*! writes the content of \c dataSource to the file \c fileName. */ void HDF5FilterPrivate::write(const QString & fileName, AbstractDataSource* dataSource) { Q_UNUSED(fileName); Q_UNUSED(dataSource); //TODO: writing HDF5 not implemented yet } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void HDF5Filter::save(QXmlStreamWriter* writer) const { writer->writeStartElement("hdfFilter"); writer->writeEndElement(); } /*! Loads from XML. */ bool HDF5Filter::load(XmlStreamReader* reader) { Q_UNUSED(reader); // KLocalizedString attributeWarning = ki18n("Attribute '%1' missing or empty, default value is used"); // QXmlStreamAttributes attribs = reader->attributes(); return true; } diff --git a/src/backend/spreadsheet/Spreadsheet.cpp b/src/backend/spreadsheet/Spreadsheet.cpp index feb272cdf..c68bd533b 100644 --- a/src/backend/spreadsheet/Spreadsheet.cpp +++ b/src/backend/spreadsheet/Spreadsheet.cpp @@ -1,959 +1,962 @@ /*************************************************************************** File : Spreadsheet.cpp Project : LabPlot Description : Aspect providing a spreadsheet table with column logic -------------------------------------------------------------------- Copyright : (C) 2006-2008 Tilman Benkert (thzs@gmx.net) Copyright : (C) 2006-2009 Knut Franke (knut.franke@gmx.de) Copyright : (C) 2012-2017 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2017 Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "Spreadsheet.h" #include "SpreadsheetModel.h" #include "backend/core/AspectPrivate.h" #include "backend/core/AbstractAspect.h" #include "backend/core/column/ColumnStringIO.h" #include "backend/core/datatypes/DateTime2StringFilter.h" #include "commonfrontend/spreadsheet/SpreadsheetView.h" #include #include #include #include #include /*! \class Spreadsheet \brief Aspect providing a spreadsheet table with column logic. Spreadsheet is a container object for columns with no data of its own. By definition, it's columns are all of its children inheriting from class Column. Thus, the basic API is already defined by AbstractAspect (managing the list of columns, notification of column insertion/removal) and Column (changing and monitoring state of the actual data). Spreadsheet stores a pointer to its primary view of class SpreadsheetView. SpreadsheetView calls the Spreadsheet API but Spreadsheet only notifies SpreadsheetView by signals without calling its API directly. This ensures a maximum independence of UI and backend. SpreadsheetView can be easily replaced by a different class. User interaction is completely handled in SpreadsheetView and translated into Spreadsheet API calls (e.g., when a user edits a cell this will be handled by the delegate of SpreadsheetView and Spreadsheet will not know whether a script or a user changed the data.). All actions, menus etc. for the user interaction are handled SpreadsheetView, e.g., via a context menu. Selections are also handled by SpreadsheetView. The view itself is created by the first call to view(); \ingroup backend */ Spreadsheet::Spreadsheet(const QString& name, bool loading) : AbstractDataSource(name) { if (!loading) init(); } /*! initializes the spreadsheet with the default number of columns and rows */ void Spreadsheet::init() { KConfigGroup group = KSharedConfig::openConfig()->group(QLatin1String("Spreadsheet")); const int columns = group.readEntry(QLatin1String("ColumnCount"), 2); const int rows = group.readEntry(QLatin1String("RowCount"), 100); for (int i = 0; i < columns; i++) { Column* new_col = new Column(QString::number(i+1), AbstractColumn::Numeric); new_col->setPlotDesignation(i == 0 ? AbstractColumn::X : AbstractColumn::Y); addChild(new_col); } setRowCount(rows); } void Spreadsheet::setModel(SpreadsheetModel* model) { m_model = model; } SpreadsheetModel * Spreadsheet::model() { return m_model; } /*! Constructs a primary view on me. This method may be called multiple times during the life time of an Aspect, or it might not get called at all. Aspects must not depend on the existence of a view for their operation. */ QWidget* Spreadsheet::view() const { if (!m_partView) { m_view = new SpreadsheetView(const_cast(this)); m_partView = m_view; } return m_partView; } bool Spreadsheet::exportView() const { return m_view->exportView(); } bool Spreadsheet::printView() { return m_view->printView(); } bool Spreadsheet::printPreview() const { return m_view->printPreview(); } /*! Returns the maximum number of rows in the spreadsheet. */ int Spreadsheet::rowCount() const { int result = 0; for (auto* col : children()) { const int col_rows = col->rowCount(); if ( col_rows > result) result = col_rows; } return result; } void Spreadsheet::removeRows(int first, int count) { if ( count < 1 || first < 0 || first+count > rowCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: remove 1 row", "%1: remove %2 rows", name(), count) ); for (auto* col : children(IncludeHidden)) col->removeRows(first, count); endMacro(); RESET_CURSOR; } void Spreadsheet::insertRows(int before, int count) { if ( count < 1 || before < 0 || before > rowCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: insert 1 row", "%1: insert %2 rows", name(), count) ); for (auto* col : children(IncludeHidden)) col->insertRows(before, count); endMacro(); RESET_CURSOR; } void Spreadsheet::appendRows(int count) { insertRows(rowCount(), count); } void Spreadsheet::appendRow() { insertRows(rowCount(), 1); } void Spreadsheet::appendColumns(int count) { insertColumns(columnCount(), count); } void Spreadsheet::appendColumn() { insertColumns(columnCount(), 1); } void Spreadsheet::prependColumns(int count) { insertColumns(0, count); } /*! Sets the number of rows of the spreadsheet to \c new_size */ void Spreadsheet::setRowCount(int new_size) { int current_size = rowCount(); if (new_size > current_size) insertRows(current_size, new_size-current_size); if (new_size < current_size && new_size >= 0) removeRows(new_size, current_size-new_size); } /*! Returns the column with the number \c index. Shallow wrapper around \sa AbstractAspect::child() - see there for caveat. */ Column* Spreadsheet::column(int index) const { return child(index); } /*! Returns the column with the name \c name. */ Column* Spreadsheet::column(const QString &name) const { return child(name); } /*! Returns the total number of columns in the spreadsheet. */ int Spreadsheet::columnCount() const { return childCount(); } /*! Returns the number of columns matching the given designation. */ int Spreadsheet::columnCount(AbstractColumn::PlotDesignation pd) const { int count = 0; for (auto* col : children()) if (col->plotDesignation() == pd) count++; return count; } void Spreadsheet::removeColumns(int first, int count) { if ( count < 1 || first < 0 || first+count > columnCount()) return; WAIT_CURSOR; beginMacro( i18np("%1: remove 1 column", "%1: remove %2 columns", name(), count) ); for (int i = 0; i < count; i++) child(first)->remove(); endMacro(); RESET_CURSOR; } void Spreadsheet::insertColumns(int before, int count) { WAIT_CURSOR; beginMacro( i18np("%1: insert 1 column", "%1: insert %2 columns", name(), count) ); Column * before_col = column(before); int rows = rowCount(); for (int i = 0; i < count; i++) { Column * new_col = new Column(QString::number(i+1), AbstractColumn::Numeric); new_col->setPlotDesignation(AbstractColumn::Y); new_col->insertRows(0, rows); insertChildBefore(new_col, before_col); } endMacro(); RESET_CURSOR; } /*! Sets the number of columns to \c new_size */ void Spreadsheet::setColumnCount(int new_size) { int old_size = columnCount(); if ( old_size == new_size || new_size < 0 ) return; if (new_size < old_size) removeColumns(new_size, old_size-new_size); else insertColumns(old_size, new_size-old_size); } /*! Clears the whole spreadsheet. */ void Spreadsheet::clear() { WAIT_CURSOR; beginMacro(i18n("%1: clear", name())); for (auto* col : children()) col->clear(); endMacro(); RESET_CURSOR; } /*! Clears all mask in the spreadsheet. */ void Spreadsheet::clearMasks() { WAIT_CURSOR; beginMacro(i18n("%1: clear all masks", name())); for (auto* col : children()) col->clearMasks(); endMacro(); RESET_CURSOR; } /*! Returns a new context menu. The caller takes ownership of the menu. */ QMenu* Spreadsheet::createContextMenu() { QMenu* menu = AbstractPart::createContextMenu(); Q_ASSERT(menu); emit requestProjectContextMenu(menu); return menu; } void Spreadsheet::moveColumn(int from, int to) { Column* col = child(from); beginMacro(i18n("%1: move column %2 from position %3 to %4.", name(), col->name(), from+1, to+1)); col->remove(); insertChildBefore(col, child(to)); endMacro(); } void Spreadsheet::copy(Spreadsheet* other) { WAIT_CURSOR; beginMacro(i18n("%1: copy %2", name(), other->name())); for (auto* col : children()) col->remove(); for (auto* src_col : other->children()) { Column * new_col = new Column(src_col->name(), src_col->columnMode()); new_col->copy(src_col); new_col->setPlotDesignation(src_col->plotDesignation()); QVector< Interval > masks = src_col->maskedIntervals(); for (const auto& iv : masks) new_col->setMasked(iv); QVector< Interval > formulas = src_col->formulaIntervals(); for (const auto& iv : formulas) new_col->setFormula(iv, src_col->formula(iv.start())); new_col->setWidth(src_col->width()); addChild(new_col); } setComment(other->comment()); endMacro(); RESET_CURSOR; } // FIXME: replace index-based API with Column*-based one /*! Determines the corresponding X column. */ int Spreadsheet::colX(int col) { for (int i = col-1; i >= 0; i--) { if (column(i)->plotDesignation() == AbstractColumn::X) return i; } int cols = columnCount(); for (int i = col+1; i < cols; i++) { if (column(i)->plotDesignation() == AbstractColumn::X) return i; } return -1; } /*! Determines the corresponding Y column. */ int Spreadsheet::colY(int col) { int cols = columnCount(); if (column(col)->plotDesignation() == AbstractColumn::XError || column(col)->plotDesignation() == AbstractColumn::YError) { // look to the left first for (int i = col-1; i >= 0; i--) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } for (int i = col+1; i < cols; i++) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } } else { // look to the right first for (int i = col+1; i < cols; i++) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } for (int i = col-1; i >= 0; i--) { if (column(i)->plotDesignation() == AbstractColumn::Y) return i; } } return -1; } /*! Sorts the given list of column. If 'leading' is a null pointer, each column is sorted separately. */ void Spreadsheet::sortColumns(Column* leading, QVector cols, bool ascending) { if (cols.isEmpty()) return; // the normal QPair comparison does not work properly with descending sorting // therefore we use our own compare functions class CompareFunctions { public: static bool doubleLess(const QPair& a, const QPair& b) { return a.first < b.first; } static bool doubleGreater(const QPair& a, const QPair& b) { return a.first > b.first; } static bool integerLess(const QPair& a, const QPair& b) { return a.first < b.first; } static bool integerGreater(const QPair& a, const QPair& b) { return a.first > b.first; } static bool QStringLess(const QPair& a, const QPair& b) { return a < b; } static bool QStringGreater(const QPair& a, const QPair& b) { return a > b; } static bool QDateTimeLess(const QPair& a, const QPair& b) { return a < b; } static bool QDateTimeGreater(const QPair& a, const QPair& b) { return a > b; } }; WAIT_CURSOR; beginMacro(i18n("%1: sort columns", name())); if (leading == nullptr) { // sort separately for (auto* col : cols) { switch (col->columnMode()) { case AbstractColumn::Numeric: { int rows = col->rowCount(); QVector< QPair > map; for (int j = 0; j < rows; j++) map.append(QPair(col->valueAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::Integer: { int rows = col->rowCount(); QVector< QPair > map; for (int j = 0; j < rows; j++) map.append(QPair(col->valueAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator> it(map); Column* temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::Text: { int rows = col->rowCount(); QVector> map; for (int j = 0; j < rows; j++) map.append(QPair(col->textAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringGreater); QVectorIterator< QPair > it(map); Column* temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { int rows = col->rowCount(); QVector< QPair > map; for (int j = 0; j < rows; j++) map.append(QPair(col->dateTimeAt(j), j)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeGreater); QVectorIterator< QPair > it(map); Column *temp_col = new Column("temp", col->columnMode()); int k = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, k, 1); temp_col->setMasked(col->isMasked(it.next().second)); k++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; break; } } } } else { // sort with leading column switch (leading->columnMode()) { case AbstractColumn::Numeric: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->valueAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::doubleGreater); QVectorIterator> it(map); for (auto* col : cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::Integer: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->valueAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::integerLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::integerGreater); QVectorIterator> it(map); for (auto* col : cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::Text: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->textAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QStringGreater); QVectorIterator> it(map); for (auto* col : cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } case AbstractColumn::DateTime: case AbstractColumn::Month: case AbstractColumn::Day: { QVector> map; int rows = leading->rowCount(); for (int i = 0; i < rows; i++) map.append(QPair(leading->dateTimeAt(i), i)); if (ascending) std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeLess); else std::stable_sort(map.begin(), map.end(), CompareFunctions::QDateTimeGreater); QVectorIterator> it(map); for (auto* col : cols) { Column *temp_col = new Column("temp", col->columnMode()); it.toFront(); int j = 0; // put the values in the right order into temp_col while (it.hasNext()) { temp_col->copy(col, it.peekNext().second, j, 1); temp_col->setMasked(col->isMasked(it.next().second)); j++; } // copy the sorted column col->copy(temp_col, 0, 0, rows); delete temp_col; } break; } } } endMacro(); RESET_CURSOR; } // end of sortColumns() /*! Returns an icon to be used for decorating my views. */ QIcon Spreadsheet::icon() const { return QIcon::fromTheme("labplot-spreadsheet"); } /*! Returns the text displayed in the given cell. */ QString Spreadsheet::text(int row, int col) const { Column* c = column(col); if (!c) return QString(); return c->asStringColumn()->textAt(row); } /*! * This slot is, indirectly, called when a child of \c Spreadsheet (i.e. column) was selected in \c ProjectExplorer. * Emits the signal \c columnSelected that is handled in \c SpreadsheetView. */ void Spreadsheet::childSelected(const AbstractAspect* aspect) { const Column* column = qobject_cast(aspect); if (column) { int index = indexOfChild(column); emit columnSelected(index); } } /*! * This slot is, indirectly, called when a child of \c Spreadsheet (i.e. column) was deselected in \c ProjectExplorer. * Emits the signal \c columnDeselected that is handled in \c SpreadsheetView. */ void Spreadsheet::childDeselected(const AbstractAspect* aspect) { const Column* column = qobject_cast(aspect); if (column) { int index = indexOfChild(column); emit columnDeselected(index); } } /*! * Emits the signal to select or to deselect the column number \c index in the project explorer, * if \c selected=true or \c selected=false, respectively. * The signal is handled in \c AspectTreeModel and forwarded to the tree view in \c ProjectExplorer. * This function is called in \c SpreadsheetView upon selection changes. */ void Spreadsheet::setColumnSelectedInView(int index, bool selected) { if (selected) { emit childAspectSelectedInView(child(index)); //deselect the spreadsheet in the project explorer, if a child (column) was selected. //prevents unwanted multiple selection with spreadsheet (if it was selected before). emit childAspectDeselectedInView(this); } else emit childAspectDeselectedInView(child(index)); } //############################################################################## //################## Serialization/Deserialization ########################### //############################################################################## /*! Saves as XML. */ void Spreadsheet::save(QXmlStreamWriter* writer) const { writer->writeStartElement("spreadsheet"); writeBasicAttributes(writer); writeCommentElement(writer); //columns for (auto* col : children(IncludeHidden)) col->save(writer); writer->writeEndElement(); // "spreadsheet" } /*! Loads from XML. */ bool Spreadsheet::load(XmlStreamReader* reader, bool preview) { if (!readBasicAttributes(reader)) return false; // read child elements while (!reader->atEnd()) { reader->readNext(); if (reader->isEndElement()) break; if (reader->isStartElement()) { if (reader->name() == "comment") { if (!readCommentElement(reader)) return false; } else if (reader->name() == "column") { Column* column = new Column(QString()); if (!column->load(reader, preview)) { delete column; setColumnCount(0); return false; } addChildFast(column); } else { // unknown element reader->raiseWarning(i18n("unknown element '%1'", reader->name().toString())); if (!reader->skipToEndElement()) return false; } } } return !reader->hasError(); } //############################################################################## //######################## Data Import ####################################### //############################################################################## int Spreadsheet::prepareImport(QVector& dataContainer, AbstractFileFilter::ImportMode importMode, int actualRows, int actualCols, QStringList colNameList, QVector columnMode) { - DEBUG("Spreadsheet::prepareImport()"); - DEBUG(" resize spreadsheet to rows = " << actualRows << " and cols = " << actualCols); + DEBUG("Spreadsheet::prepareImport()") + DEBUG(" resize spreadsheet to rows = " << actualRows << " and cols = " << actualCols) + QDEBUG(" column name list = " << colNameList) int columnOffset = 0; setUndoAware(false); if (m_model != nullptr) m_model->suppressSignals(true); //make the available columns undo unaware before we resize and rename them below, //the same will be done for new columns in this->resize(). for (int i = 0; i < childCount(); i++) child(i)->setUndoAware(false); columnOffset = this->resize(importMode, colNameList, actualCols); // resize the spreadsheet if (importMode == AbstractFileFilter::Replace) { clear(); setRowCount(actualRows); } else { if (rowCount() < actualRows) setRowCount(actualRows); } if (columnMode.size() < actualCols) { qWarning("columnMode[] size is too small! Giving up."); return -1; } dataContainer.resize(actualCols); for (int n = 0; n < actualCols; n++) { // data() returns a void* which is a pointer to any data type (see ColumnPrivate.cpp) Column* column = this->child(columnOffset+n); DEBUG(" column " << n << " columnMode = " << columnMode[n]); column->setColumnMode(columnMode[n]); //in the most cases the first imported column is meant to be used as x-data. //Other columns provide mostly y-data or errors. //TODO: this has to be configurable for the user in the import widget, //it should be possible to specify x-error plot designation, etc. AbstractColumn::PlotDesignation desig = (n == 0) ? AbstractColumn::X : AbstractColumn::Y; column->setPlotDesignation(desig); switch (columnMode[n]) { case AbstractColumn::Numeric: { auto* vector = static_cast*>(column->data()); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Integer: { auto* vector = static_cast*>(column->data()); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Text: { auto* vector = static_cast*>(column->data()); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } case AbstractColumn::Month: case AbstractColumn::Day: case AbstractColumn::DateTime: { auto* vector = static_cast* >(column->data()); vector->resize(actualRows); dataContainer[n] = static_cast(vector); break; } } } // QDEBUG("dataPointers =" << dataPointers); DEBUG("Spreadsheet::prepareImport() DONE"); return columnOffset; } /*! resize data source to cols columns returns column offset depending on import mode */ int Spreadsheet::resize(AbstractFileFilter::ImportMode mode, QStringList colNameList, int cols) { + DEBUG("Spreadsheet::resize()") + QDEBUG(" column name list = " << colNameList) // name additional columns for (int k = colNameList.size(); k < cols; k++ ) colNameList.append( "Column " + QString::number(k+1) ); int columnOffset = 0; //indexes the "start column" in the spreadsheet. Starting from this column the data will be imported. Column* newColumn = nullptr; if (mode == AbstractFileFilter::Append) { columnOffset = childCount(); for (int n = 0; n < cols; n++ ) { newColumn = new Column(colNameList.at(n), AbstractColumn::Numeric); newColumn->setUndoAware(false); addChildFast(newColumn); } } else if (mode == AbstractFileFilter::Prepend) { Column* firstColumn = child(0); for (int n = 0; n < cols; n++ ) { newColumn = new Column(colNameList.at(n), AbstractColumn::Numeric); newColumn->setUndoAware(false); insertChildBeforeFast(newColumn, firstColumn); } } else if (mode == AbstractFileFilter::Replace) { //replace completely the previous content of the data source with the content to be imported. int columns = childCount(); if (columns > cols) { //there're more columns in the data source then required -> remove the superfluous columns for (int i = 0; i < columns-cols; i++) removeChild(child(0)); } else { //create additional columns if needed for (int i = columns; i < cols; i++) { newColumn = new Column(colNameList.at(i), AbstractColumn::Numeric); newColumn->setUndoAware(false); addChildFast(newColumn); } } //rename the columns that are already available and suppress the dataChanged signal for them for (int i = 0; i < childCount(); i++) { if (mode == AbstractFileFilter::Replace) child(i)->setSuppressDataChangedSignal(true); child(i)->setName(colNameList.at(i)); } } return columnOffset; } void Spreadsheet::finalizeImport(int columnOffset, int startColumn, int endColumn, int numRows, const QString& dateTimeFormat, AbstractFileFilter::ImportMode importMode) { DEBUG("Spreadsheet::finalizeImport()"); // shrink the spreadsheet if needed if (numRows > 0 && numRows != rowCount()) { if (importMode == AbstractFileFilter::Replace) setRowCount(numRows); } // set the comments for each of the columns if datasource is a spreadsheet const int rows = rowCount(); for (int n = startColumn; n <= endColumn; n++) { Column* column = this->column(columnOffset + n - startColumn); DEBUG(" column " << n << " of type " << column->columnMode()); QString comment; switch (column->columnMode()) { case AbstractColumn::Numeric: comment = i18np("numerical data, %1 element", "numerical data, %1 elements", rows); break; case AbstractColumn::Integer: comment = i18np("integer data, %1 element", "integer data, %1 elements", rows); break; case AbstractColumn::Text: comment = i18np("text data, %1 element", "text data, %1 elements", rows); break; case AbstractColumn::Month: comment = i18np("month data, %1 element", "month data, %1 elements", rows); break; case AbstractColumn::Day: comment = i18np("day data, %1 element", "day data, %1 elements", rows); break; case AbstractColumn::DateTime: comment = i18np("date and time data, %1 element", "date and time data, %1 elements", rows); // set same datetime format in column auto* filter = static_cast(column->outputFilter()); filter->setFormat(dateTimeFormat); } column->setComment(comment); if (importMode == AbstractFileFilter::Replace) { column->setSuppressDataChangedSignal(false); column->setChanged(); } } //make the spreadsheet and all its children undo aware again setUndoAware(true); for (int i = 0; i < childCount(); i++) child(i)->setUndoAware(true); if (m_model != nullptr) m_model->suppressSignals(false); if (m_partView != nullptr && m_view != nullptr) m_view->resizeHeader(); DEBUG("Spreadsheet::finalizeImport() DONE"); } diff --git a/src/kdefrontend/datasources/ImportFileDialog.cpp b/src/kdefrontend/datasources/ImportFileDialog.cpp index a0cec75d9..86af3866a 100644 --- a/src/kdefrontend/datasources/ImportFileDialog.cpp +++ b/src/kdefrontend/datasources/ImportFileDialog.cpp @@ -1,491 +1,491 @@ /*************************************************************************** File : ImportDialog.cc Project : LabPlot Description : import file data dialog -------------------------------------------------------------------- Copyright : (C) 2008-2019 Alexander Semke (alexander.semke@web.de) Copyright : (C) 2008-2015 by Stefan Gerlach (stefan.gerlach@uni.kn) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "ImportFileDialog.h" #include "ImportFileWidget.h" #include "backend/core/AspectTreeModel.h" #include "backend/datasources/LiveDataSource.h" #include "backend/datasources/filters/AbstractFileFilter.h" #include "backend/datasources/filters/filters.h" #include "backend/spreadsheet/Spreadsheet.h" #include "backend/matrix/Matrix.h" #include "backend/core/Workbook.h" #include "commonfrontend/widgets/TreeViewComboBox.h" #include "kdefrontend/MainWin.h" #ifdef HAVE_MQTT #include "backend/datasources/MQTTClient.h" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*! \class ImportFileDialog \brief Dialog for importing data from a file. Embeds \c ImportFileWidget and provides the standard buttons. \ingroup kdefrontend */ ImportFileDialog::ImportFileDialog(MainWin* parent, bool liveDataSource, const QString& fileName) : ImportDialog(parent), m_importFileWidget(new ImportFileWidget(this, liveDataSource, fileName)) { vLayout->addWidget(m_importFileWidget); //dialog buttons QDialogButtonBox* buttonBox = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Reset |QDialogButtonBox::Cancel); okButton = buttonBox->button(QDialogButtonBox::Ok); m_optionsButton = buttonBox->button(QDialogButtonBox::Reset); //we highjack the default "Reset" button and use if for showing/hiding the options okButton->setEnabled(false); //ok is only available if a valid container was selected vLayout->addWidget(buttonBox); //hide the data-source related widgets if (!liveDataSource) setModel(); //Signals/Slots connect(buttonBox, &QDialogButtonBox::accepted, this, &QDialog::accept); connect(buttonBox, &QDialogButtonBox::rejected, this, &QDialog::reject); if (!liveDataSource) setWindowTitle(i18nc("@title:window", "Import Data to Spreadsheet or Matrix")); else setWindowTitle(i18nc("@title:window", "Add New Live Data Source")); setWindowIcon(QIcon::fromTheme("document-import-database")); //restore saved settings if available create(); // ensure there's a window created QApplication::processEvents(QEventLoop::AllEvents, 0); m_importFileWidget->loadSettings(); KConfigGroup conf(KSharedConfig::openConfig(), "ImportFileDialog"); if (conf.exists()) { m_showOptions = conf.readEntry("ShowOptions", false); m_showOptions ? m_optionsButton->setText(i18n("Hide Options")) : m_optionsButton->setText(i18n("Show Options")); m_importFileWidget->showOptions(m_showOptions); //do the signal-slot connections after all settings were loaded in import file widget and check the OK button after this connect(m_importFileWidget, &ImportFileWidget::checkedFitsTableToMatrix, this, &ImportFileDialog::checkOnFitsTableToMatrix); connect(m_importFileWidget, static_cast(&ImportFileWidget::fileNameChanged), this, &ImportFileDialog::checkOkButton); connect(m_importFileWidget, static_cast(&ImportFileWidget::sourceTypeChanged), this, &ImportFileDialog::checkOkButton); connect(m_importFileWidget, &ImportFileWidget::hostChanged, this, &ImportFileDialog::checkOkButton); connect(m_importFileWidget, &ImportFileWidget::portChanged, this, &ImportFileDialog::checkOkButton); //TODO: do we really need to check the ok button when the preview was refreshed? //If not, remove this together with the previewRefreshed signal in ImportFileWidget //connect(m_importFileWidget, &ImportFileWidget::previewRefreshed, this, &ImportFileDialog::checkOkButton); #ifdef HAVE_MQTT connect(m_importFileWidget, &ImportFileWidget::subscriptionsChanged, this, &ImportFileDialog::checkOkButton); connect(m_importFileWidget, &ImportFileWidget::checkFileType, this, &ImportFileDialog::checkOkButton); #endif connect(m_optionsButton, &QPushButton::clicked, this, &ImportFileDialog::toggleOptions); KWindowConfig::restoreWindowSize(windowHandle(), conf); resize(windowHandle()->size()); // workaround for QTBUG-40584 } else resize(QSize(0, 0).expandedTo(minimumSize())); checkOkButton(); } ImportFileDialog::~ImportFileDialog() { //save current settings KConfigGroup conf(KSharedConfig::openConfig(), "ImportFileDialog"); conf.writeEntry("ShowOptions", m_showOptions); if (cbPosition) conf.writeEntry("Position", cbPosition->currentIndex()); KWindowConfig::saveWindowSize(windowHandle(), conf); } int ImportFileDialog::sourceType() const { return static_cast(m_importFileWidget->currentSourceType()); } /*! triggers data import to the live data source \c source */ void ImportFileDialog::importToLiveDataSource(LiveDataSource* source, QStatusBar* statusBar) const { DEBUG("ImportFileDialog::importToLiveDataSource()"); m_importFileWidget->saveSettings(source); //show a progress bar in the status bar auto* progressBar = new QProgressBar(); progressBar->setRange(0, 100); connect(source->filter(), &AbstractFileFilter::completed, progressBar, &QProgressBar::setValue); statusBar->clearMessage(); statusBar->addWidget(progressBar, 1); WAIT_CURSOR; QTime timer; timer.start(); DEBUG(" Initial read()"); source->read(); statusBar->showMessage( i18n("Live data source created in %1 seconds.", (float)timer.elapsed()/1000) ); RESET_CURSOR; statusBar->removeWidget(progressBar); source->ready(); } #ifdef HAVE_MQTT /*! triggers data import to the MQTTClient \c client */ void ImportFileDialog::importToMQTT(MQTTClient* client) const{ m_importFileWidget->saveMQTTSettings(client); client->read(); client->ready(); } #endif /*! triggers data import to the currently selected data container */ void ImportFileDialog::importTo(QStatusBar* statusBar) const { DEBUG("ImportFileDialog::importTo()"); QDEBUG(" cbAddTo->currentModelIndex() =" << cbAddTo->currentModelIndex()); AbstractAspect* aspect = static_cast(cbAddTo->currentModelIndex().internalPointer()); if (!aspect) { DEBUG("ERROR in importTo(): No aspect available"); DEBUG(" cbAddTo->currentModelIndex().isValid() = " << cbAddTo->currentModelIndex().isValid()); DEBUG(" cbAddTo->currentModelIndex() row/column = " << cbAddTo->currentModelIndex().row() << ' ' << cbAddTo->currentModelIndex().column()); return; } if (m_importFileWidget->isFileEmpty()) { KMessageBox::information(nullptr, i18n("No data to import."), i18n("No Data")); return; } QString fileName = m_importFileWidget->fileName(); auto filter = m_importFileWidget->currentFileFilter(); auto mode = AbstractFileFilter::ImportMode(cbPosition->currentIndex()); //show a progress bar in the status bar auto* progressBar = new QProgressBar(); progressBar->setRange(0, 100); connect(filter, &AbstractFileFilter::completed, progressBar, &QProgressBar::setValue); statusBar->clearMessage(); statusBar->addWidget(progressBar, 1); WAIT_CURSOR; QApplication::processEvents(QEventLoop::AllEvents, 100); QTime timer; timer.start(); if (aspect->inherits("Matrix")) { - DEBUG(" to Matrix"); + DEBUG("ImportFileDialog::importTo(): to Matrix"); auto* matrix = qobject_cast(aspect); filter->readDataFromFile(fileName, matrix, mode); } else if (aspect->inherits("Spreadsheet")) { - DEBUG(" to Spreadsheet"); + DEBUG("ImportFileDialog::importTo(): to Spreadsheet"); auto* spreadsheet = qobject_cast(aspect); - DEBUG(" Calling readDataFromFile() with spreadsheet " << spreadsheet); + DEBUG(" Calling filter->readDataFromFile() with spreadsheet " << spreadsheet); filter->readDataFromFile(fileName, spreadsheet, mode); } else if (aspect->inherits("Workbook")) { - DEBUG(" to Workbook"); + DEBUG("ImportFileDialog::importTo(): to Workbook"); auto* workbook = qobject_cast(aspect); QVector sheets = workbook->children(); AbstractFileFilter::FileType fileType = m_importFileWidget->currentFileType(); // multiple data sets/variables for HDF5, NetCDF and ROOT if (fileType == AbstractFileFilter::HDF5 || fileType == AbstractFileFilter::NETCDF || fileType == AbstractFileFilter::ROOT) { QStringList names; if (fileType == AbstractFileFilter::HDF5) names = m_importFileWidget->selectedHDF5Names(); else if (fileType == AbstractFileFilter::NETCDF) names = m_importFileWidget->selectedNetCDFNames(); else names = m_importFileWidget->selectedROOTNames(); int nrNames = names.size(), offset = sheets.size(); //TODO: think about importing multiple sets into one sheet int start = 0; // add nrNames sheets (0 to nrNames) if (mode == AbstractFileFilter::Replace) // add only missing sheets (from offset to nrNames) start = offset; // add additional sheets for (int i = start; i < nrNames; ++i) { Spreadsheet *spreadsheet = new Spreadsheet(i18n("Spreadsheet")); if (mode == AbstractFileFilter::Prepend) workbook->insertChildBefore(spreadsheet, sheets[0]); else workbook->addChild(spreadsheet); } // start at offset for append, else at 0 if (mode != AbstractFileFilter::Append) offset = 0; // import all sets to a different sheet sheets = workbook->children(); for (int i = 0; i < nrNames; ++i) { if (fileType == AbstractFileFilter::HDF5) static_cast(filter)->setCurrentDataSetName(names[i]); else if (fileType == AbstractFileFilter::NETCDF) static_cast(filter)->setCurrentVarName(names[i]); else static_cast(filter)->setCurrentObject(names[i]); int index = i + offset; if (sheets[index]->inherits("Matrix")) filter->readDataFromFile(fileName, qobject_cast(sheets[index])); else if (sheets[index]->inherits("Spreadsheet")) filter->readDataFromFile(fileName, qobject_cast(sheets[index])); } } else { // single import file types // use active spreadsheet/matrix if present, else new spreadsheet Spreadsheet* spreadsheet = workbook->currentSpreadsheet(); Matrix* matrix = workbook->currentMatrix(); if (spreadsheet) filter->readDataFromFile(fileName, spreadsheet, mode); else if (matrix) filter->readDataFromFile(fileName, matrix, mode); else { spreadsheet = new Spreadsheet(i18n("Spreadsheet")); workbook->addChild(spreadsheet); filter->readDataFromFile(fileName, spreadsheet, mode); } } } statusBar->showMessage(i18n("File %1 imported in %2 seconds.", fileName, (float)timer.elapsed()/1000)); RESET_CURSOR; statusBar->removeWidget(progressBar); } void ImportFileDialog::toggleOptions() { m_importFileWidget->showOptions(!m_showOptions); m_showOptions = !m_showOptions; m_showOptions ? m_optionsButton->setText(i18n("Hide Options")) : m_optionsButton->setText(i18n("Show Options")); //resize the dialog layout()->activate(); resize( QSize(this->width(), 0).expandedTo(minimumSize()) ); } void ImportFileDialog::checkOnFitsTableToMatrix(const bool enable) { if (cbAddTo) { QDEBUG("cbAddTo->currentModelIndex() = " << cbAddTo->currentModelIndex()); AbstractAspect* aspect = static_cast(cbAddTo->currentModelIndex().internalPointer()); if (!aspect) { DEBUG("ERROR: no aspect available."); return; } if (aspect->inherits("Matrix")) { okButton->setEnabled(enable); if (enable) okButton->setToolTip(i18n("Close the dialog and import the data.")); else okButton->setToolTip(i18n("Cannot import into a matrix since the data contains non-numerical data.")); } } } void ImportFileDialog::checkOkButton() { DEBUG("ImportFileDialog::checkOkButton()"); if (cbAddTo) { //only check for the target container when no file data source is being added QDEBUG(" cbAddTo->currentModelIndex() = " << cbAddTo->currentModelIndex()); AbstractAspect* aspect = static_cast(cbAddTo->currentModelIndex().internalPointer()); if (!aspect) { okButton->setEnabled(false); okButton->setToolTip(i18n("Select a data container where the data has to be imported into.")); lPosition->setEnabled(false); cbPosition->setEnabled(false); return; } else { lPosition->setEnabled(true); cbPosition->setEnabled(true); //when doing ASCII import to a matrix, hide the options for using the file header (first line) //to name the columns since the column names are fixed in a matrix const auto* matrix = dynamic_cast(aspect); m_importFileWidget->showAsciiHeaderOptions(matrix == nullptr); } } QString fileName = m_importFileWidget->fileName(); #ifndef HAVE_WINDOWS if (!fileName.isEmpty() && fileName.at(0) != QDir::separator()) fileName = QDir::homePath() + QDir::separator() + fileName; #endif DEBUG("Data Source Type: " << ENUM_TO_STRING(LiveDataSource, SourceType, m_importFileWidget->currentSourceType())); switch (m_importFileWidget->currentSourceType()) { case LiveDataSource::SourceType::FileOrPipe: { DEBUG("fileName = " << fileName.toUtf8().constData()); const bool enable = QFile::exists(fileName); okButton->setEnabled(enable); if (enable) okButton->setToolTip(i18n("Close the dialog and import the data.")); else okButton->setToolTip(i18n("Provide an existing file.")); break; } case LiveDataSource::SourceType::LocalSocket: { const bool enable = QFile::exists(fileName); if (enable) { QLocalSocket lsocket{this}; DEBUG("CONNECT"); lsocket.connectToServer(fileName, QLocalSocket::ReadOnly); if (lsocket.waitForConnected()) { // this is required for server that send data as soon as connected lsocket.waitForReadyRead(); DEBUG("DISCONNECT"); lsocket.disconnectFromServer(); // read-only socket is disconnected immediately (no waitForDisconnected()) okButton->setEnabled(true); okButton->setToolTip(i18n("Close the dialog and import the data.")); } else { DEBUG("failed connect to local socket - " << lsocket.errorString().toStdString()); okButton->setEnabled(false); okButton->setToolTip(i18n("Could not connect to the provided local socket.")); } } else { okButton->setEnabled(false); okButton->setToolTip(i18n("Selected local socket does not exist.")); } break; } case LiveDataSource::SourceType::NetworkTcpSocket: { const bool enable = !m_importFileWidget->host().isEmpty() && !m_importFileWidget->port().isEmpty(); if (enable) { QTcpSocket socket(this); socket.connectToHost(m_importFileWidget->host(), m_importFileWidget->port().toUShort(), QTcpSocket::ReadOnly); if (socket.waitForConnected()) { okButton->setEnabled(true); okButton->setToolTip(i18n("Close the dialog and import the data.")); socket.disconnectFromHost(); } else { DEBUG("failed to connect to TCP socket - " << socket.errorString().toStdString()); okButton->setEnabled(false); okButton->setToolTip(i18n("Could not connect to the provided TCP socket.")); } } else { okButton->setEnabled(false); okButton->setToolTip(i18n("Either the host name or the port number is missing.")); } break; } case LiveDataSource::SourceType::NetworkUdpSocket: { const bool enable = !m_importFileWidget->host().isEmpty() && !m_importFileWidget->port().isEmpty(); if (enable) { QUdpSocket socket(this); socket.bind(QHostAddress(m_importFileWidget->host()), m_importFileWidget->port().toUShort()); socket.connectToHost(m_importFileWidget->host(), 0, QUdpSocket::ReadOnly); if (socket.waitForConnected()) { okButton->setEnabled(true); okButton->setToolTip(i18n("Close the dialog and import the data.")); socket.disconnectFromHost(); // read-only socket is disconnected immediately (no waitForDisconnected()) } else { DEBUG("failed to connect to UDP socket - " << socket.errorString().toStdString()); okButton->setEnabled(false); okButton->setToolTip(i18n("Could not connect to the provided UDP socket.")); } } else { okButton->setEnabled(false); okButton->setToolTip(i18n("Either the host name or the port number is missing.")); } break; } case LiveDataSource::SourceType::SerialPort: { const QString sPort = m_importFileWidget->serialPort(); const int baudRate = m_importFileWidget->baudRate(); if (!sPort.isEmpty()) { QSerialPort serialPort{this}; DEBUG(" Port: " << sPort.toStdString() << ", Settings: " << baudRate << ',' << serialPort.dataBits() << ',' << serialPort.parity() << ',' << serialPort.stopBits()); serialPort.setPortName(sPort); serialPort.setBaudRate(baudRate); const bool serialPortOpened = serialPort.open(QIODevice::ReadOnly); okButton->setEnabled(serialPortOpened); if (serialPortOpened) { okButton->setToolTip(i18n("Close the dialog and import the data.")); serialPort.close(); } else { DEBUG("Could not connect to the provided serial port"); okButton->setToolTip(i18n("Could not connect to the provided serial port.")); } } else { okButton->setEnabled(false); okButton->setToolTip(i18n("Serial port number is missing.")); } break; } case LiveDataSource::SourceType::MQTT: { #ifdef HAVE_MQTT const bool enable = m_importFileWidget->isMqttValid(); if (enable) { okButton->setEnabled(true); okButton->setToolTip(i18n("Close the dialog and import the data.")); } else { okButton->setEnabled(false); okButton->setToolTip(i18n("Either there is no connection, or no subscriptions were made, or the file filter is not ASCII.")); } #endif break; } } } QString ImportFileDialog::selectedObject() const { return m_importFileWidget->selectedObject(); }