diff --git a/src/backend/datasources/DatasetHandler.cpp b/src/backend/datasources/DatasetHandler.cpp index 527b9bfbf..2967f738d 100644 --- a/src/backend/datasources/DatasetHandler.cpp +++ b/src/backend/datasources/DatasetHandler.cpp @@ -1,269 +1,278 @@ /*************************************************************************** File : DatasetHandler.cpp Project : LabPlot Description : Processes a dataset's metadata file -------------------------------------------------------------------- Copyright : (C) 2019 Kovacs Ferencz (kferike98@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "backend/datasources/filters/AsciiFilter.h" #include "backend/datasources/DatasetHandler.h" #include #include #include #include #include #include #include #include #include #include #include DatasetHandler::DatasetHandler(Spreadsheet* spreadsheet) : m_spreadsheet(spreadsheet), m_filter(new AsciiFilter), m_document(new QJsonDocument), m_downloadManager(new QNetworkAccessManager) { connect(m_downloadManager, &QNetworkAccessManager::finished, this, &DatasetHandler::downloadFinished); connect(this, &DatasetHandler::downloadCompleted, this, &DatasetHandler::processDataset); } DatasetHandler::~DatasetHandler() { delete m_document; delete m_downloadManager; delete m_filter; } void DatasetHandler::processMetadata(const QString& path) { qDebug("Start processing dataset..."); loadJsonDocument(path); m_containingDir = path.left(path.lastIndexOf(QDir::separator())); qDebug() << m_containingDir; if(!m_document->isEmpty()) { configureFilter(); configureSpreadsheet(); prepareForDataset(); } } void DatasetHandler::loadJsonDocument(const QString& path) { qDebug("Load Json document for metadata"); QFile file(path); if (file.open(QIODevice::ReadOnly)) { m_document = new QJsonDocument(QJsonDocument::fromJson(file.readAll())); file.close(); } else { qDebug("Couldn't open dataset category file"); } } void DatasetHandler::markMetadataAsInvalid() { m_invalidMetadataFile = true; QMessageBox::critical(0, "Invalid metadata file", "The metadata file for the choosen dataset is invalid!"); } void DatasetHandler::configureFilter() { qDebug("Configure filter"); if(m_document->isObject()) { QJsonObject jsonObject = m_document->object(); if(jsonObject.contains("separator")) m_filter->setSeparatingCharacter( jsonObject.value("separator").toString()); else markMetadataAsInvalid(); if(jsonObject.contains("comment_character")) m_filter->setCommentCharacter(jsonObject.value("comment_character").toString()); else markMetadataAsInvalid(); if(jsonObject.contains("create_index_column")) m_filter->setCreateIndexEnabled(jsonObject.value("create_index_column").toBool()); else markMetadataAsInvalid(); if(jsonObject.contains("skip_empty_parts")) m_filter->setSkipEmptyParts(jsonObject.value("skip_empty_parts").toBool()); else markMetadataAsInvalid(); if(jsonObject.contains("simplify_whitespaces")) m_filter->setSimplifyWhitespacesEnabled(jsonObject.value("simplify_whitespaces").toBool()); else markMetadataAsInvalid(); if(jsonObject.contains("remove_quotes")) m_filter->setRemoveQuotesEnabled(jsonObject.value("remove_quotes").toBool()); else markMetadataAsInvalid(); if(jsonObject.contains("use_first_row_for_vectorname")) m_filter->setHeaderEnabled(jsonObject.value("use_first_row_for_vectorname").toBool()); else markMetadataAsInvalid(); if(jsonObject.contains("number_format")) m_filter->setNumberFormat(QLocale::Language(jsonObject.value("number_format").toInt())); else markMetadataAsInvalid(); if(jsonObject.contains("DateTime_format")) m_filter->setDateTimeFormat(jsonObject.value("DateTime_format").toString()); else markMetadataAsInvalid(); } else { markMetadataAsInvalid(); } } void DatasetHandler::configureSpreadsheet() { qDebug("Conf spreadsheet"); if(m_document->isObject()) { const QJsonObject& jsonObject = m_document->object(); if(jsonObject.contains("name")) - m_spreadsheet->setName( jsonObject.value("name").toString()); + m_spreadsheet->setName( jsonObject.value("name").toString()); else markMetadataAsInvalid(); if(jsonObject.contains("description")) m_spreadsheet->setComment(jsonObject.value("description").toString()); } else { markMetadataAsInvalid(); } } void DatasetHandler::prepareForDataset() { qDebug("Start downloading dataset"); if(m_document->isObject()) { const QJsonObject& jsonObject = m_document->object(); if(jsonObject.contains("download")) { const QString& url = jsonObject.value("download").toString(); const QUrl downloadUrl = QUrl::fromEncoded(url.toLocal8Bit()); doDownload(url); } else { QMessageBox::critical(0, i18n("Invalid metadata file"), i18n("There is no download URL present in the metadata file!")); } } else { markMetadataAsInvalid(); } } void DatasetHandler::doDownload(const QUrl& url) { qDebug("Download request"); QNetworkRequest request(url); m_currentDownload = m_downloadManager->get(request); + connect(m_currentDownload, &QNetworkReply::downloadProgress, [this] (qint64 bytesReceived, qint64 bytesTotal) { + double progress; + if (bytesTotal == -1) + progress = 0; + else + progress = 100 * ((double) bytesReceived / (double) bytesTotal); + qDebug() << "Progress: " << progress; + emit downloadProgress(progress); + }); } void DatasetHandler::downloadFinished(QNetworkReply* reply) { qDebug("Download finished"); const QUrl& url = reply->url(); if (reply->error()) { qDebug("Download of %s failed: %s\n", url.toEncoded().constData(), qPrintable(reply->errorString())); } else { if (isHttpRedirect(reply)) { qDebug("Request was redirected.\n"); } else { QString filename = saveFileName(url); if (saveToDisk(filename, reply)) { qDebug("Download of %s succeeded (saved to %s)\n", url.toEncoded().constData(), qPrintable(filename)); m_fileName = filename; emit downloadCompleted(); } } } m_currentDownload = nullptr; reply->deleteLater(); } bool DatasetHandler::isHttpRedirect(QNetworkReply* reply) { const int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); // TODO enum/defines for status codes ? return statusCode == 301 || statusCode == 302 || statusCode == 303 || statusCode == 305 || statusCode == 307 || statusCode == 308; } QString DatasetHandler::saveFileName(const QUrl& url) { const QString path = url.path(); QString basename = QFileInfo(path).fileName(); if (basename.isEmpty()) basename = "download"; QString fileName = m_containingDir + QDir::separator() + basename; if (QFile::exists(fileName)) { // already exists, don't overwrite int i = 0; fileName += '.'; while (QFile::exists(fileName + QString::number(i))) ++i; fileName += QString::number(i); } return fileName; } bool DatasetHandler::saveToDisk(const QString& filename, QIODevice* data) { QFile file(filename); if (!file.open(QIODevice::WriteOnly)) { qDebug("Could not open %s for writing: %s\n", qPrintable(filename), qPrintable(file.errorString())); return false; } file.write(data->readAll()); file.close(); return true; } void DatasetHandler::processDataset() { m_filter->readDataFromFile(m_fileName, m_spreadsheet); configureColumns(); } void DatasetHandler::configureColumns() { if(m_document->isObject()) { const QJsonObject jsonObject = m_document->object(); int index = 0; const int columnsCount = m_spreadsheet->columnCount(); while(jsonObject.contains(i18n("column_description_%1", index)) && (index < columnsCount)) { m_spreadsheet->column(index)->setComment(jsonObject.value(i18n("column_description_%1", index)).toString()); ++index; } } else { qDebug("Invalid Json document"); } } diff --git a/src/backend/datasources/DatasetHandler.h b/src/backend/datasources/DatasetHandler.h index de37d6464..59c248a49 100644 --- a/src/backend/datasources/DatasetHandler.h +++ b/src/backend/datasources/DatasetHandler.h @@ -1,76 +1,77 @@ /*************************************************************************** File : DatasetHandler.h Project : LabPlot Description : Processes a dataset's metadata file -------------------------------------------------------------------- Copyright : (C) 2019 Kovacs Ferencz (kferike98@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef DATASETHANDLER_H #define DATASETHANDLER_H #include #include "backend/spreadsheet/Spreadsheet.h" class QJsonDocument; class AsciiFilter; class QNetworkAccessManager; class QNetworkReply; class DatasetHandler: public QObject{ Q_OBJECT public: DatasetHandler(Spreadsheet* spreadsheet); ~DatasetHandler(); void processMetadata(const QString& path); private: Spreadsheet* m_spreadsheet; AsciiFilter* m_filter; QJsonDocument* m_document; QNetworkAccessManager* m_downloadManager; QNetworkReply* m_currentDownload; QString m_fileName; bool m_invalidMetadataFile{false}; QString m_containingDir; void loadJsonDocument(const QString& path); void configureFilter(); void configureSpreadsheet(); void prepareForDataset(); void processDataset(); void doDownload(const QUrl &url); bool isHttpRedirect(QNetworkReply *reply); QString saveFileName(const QUrl &url); bool saveToDisk(const QString &filename, QIODevice *data); void markMetadataAsInvalid(); void configureColumns(); private slots: void downloadFinished(QNetworkReply *reply); signals: void downloadCompleted(); + void downloadProgress(int progress); }; #endif // DATASETHANDLER_H diff --git a/src/kdefrontend/datasources/ImportDatasetDialog.cpp b/src/kdefrontend/datasources/ImportDatasetDialog.cpp index 7287f5478..c482a3ce4 100644 --- a/src/kdefrontend/datasources/ImportDatasetDialog.cpp +++ b/src/kdefrontend/datasources/ImportDatasetDialog.cpp @@ -1,113 +1,114 @@ /*************************************************************************** File : ImportDatasetDialog.cpp Project : LabPlot Description : import dataset data dialog -------------------------------------------------------------------- Copyright : (C) 2019 Ferencz Koovacs (kferike98@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "ImportDatasetDialog.h" #include "ImportDatasetWidget.h" #include "backend/datasources/DatasetHandler.h" #include "KConfigGroup" #include "KSharedConfig" #include "KWindowConfig" #include "QWindow" #include "QProgressBar" #include "QDialogButtonBox" #include "QStatusBar" #include "QDebug" ImportDatasetDialog::ImportDatasetDialog(MainWin* parent, const QString& fileName) : ImportDialog(parent), m_importDatasetWidget(new ImportDatasetWidget(this)){ qDebug("add dataset widget"); vLayout->addWidget(m_importDatasetWidget); qDebug("Add completed"); connect(m_importDatasetWidget, &ImportDatasetWidget::datasetSelected, this, &ImportDatasetDialog::checkOkButton); //dialog buttons QDialogButtonBox* buttonBox = new QDialogButtonBox(QDialogButtonBox::Ok |QDialogButtonBox::Cancel); okButton = buttonBox->button(QDialogButtonBox::Ok); okButton->setEnabled(false); //ok is only available if a valid container was selected vLayout->addWidget(buttonBox); connect(buttonBox, &QDialogButtonBox::accepted, this, &QDialog::accept); connect(buttonBox, &QDialogButtonBox::rejected, this, &QDialog::reject); setWindowTitle(i18nc("@title:window", "Add new Dataset")); create(); QApplication::processEvents(QEventLoop::AllEvents, 0); KConfigGroup conf(KSharedConfig::openConfig(), "ImportDatasetDialog"); if (conf.exists()) { KWindowConfig::restoreWindowSize(windowHandle(), conf); resize(windowHandle()->size()); } else resize(QSize(0, 0).expandedTo(minimumSize())); checkOkButton(); } ImportDatasetDialog::~ImportDatasetDialog() { KConfigGroup conf(KSharedConfig::openConfig(), "ImportDatasetDialog"); KWindowConfig::saveWindowSize(windowHandle(), conf); } QString ImportDatasetDialog::selectedObject() const { return QString(); //return m_importDatasetWidget->selectedObject(); } void ImportDatasetDialog::importToDataset(DatasetHandler* datasetHandler, QStatusBar* statusBar) const { - m_importDatasetWidget->loadDatasetToProcess(datasetHandler); -} - -void ImportDatasetDialog::checkOkButton() { - bool enable = (!m_importDatasetWidget->getSelectedDataset().isEmpty()); - okButton->setEnabled(enable); -} - -void ImportDatasetDialog::importTo(QStatusBar* statusBar) const { - auto filter = m_importDatasetWidget->currentFileFilter(); //show a progress bar in the status bar auto* progressBar = new QProgressBar(); progressBar->setRange(0, 100); - connect(filter, &AbstractFileFilter::completed, progressBar, &QProgressBar::setValue); + connect(datasetHandler, &DatasetHandler::downloadProgress, progressBar, &QProgressBar::setValue); statusBar->clearMessage(); statusBar->addWidget(progressBar, 1); WAIT_CURSOR; QApplication::processEvents(QEventLoop::AllEvents, 100); QTime timer; timer.start(); + m_importDatasetWidget->loadDatasetToProcess(datasetHandler); + statusBar->showMessage(i18n("Dataset imported in %1 seconds.", (float)timer.elapsed()/1000)); RESET_CURSOR; statusBar->removeWidget(progressBar); } +void ImportDatasetDialog::checkOkButton() { + bool enable = (!m_importDatasetWidget->getSelectedDataset().isEmpty()); + okButton->setEnabled(enable); +} + +void ImportDatasetDialog::importTo(QStatusBar* statusBar) const { + +} + diff --git a/src/kdefrontend/datasources/ImportDatasetWidget.cpp b/src/kdefrontend/datasources/ImportDatasetWidget.cpp index fa7487e13..04717ce5a 100644 --- a/src/kdefrontend/datasources/ImportDatasetWidget.cpp +++ b/src/kdefrontend/datasources/ImportDatasetWidget.cpp @@ -1,487 +1,483 @@ /*************************************************************************** File : ImportDatasetWidget.cpp Project : LabPlot Description : import online dataset widget -------------------------------------------------------------------- Copyright : (C) 2019 Kovacs Ferencz (kferike98@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #include "src/backend/datasources/DatasetHandler.h" #include "src/kdefrontend/datasources/ImportDatasetWidget.h" #include "src/kdefrontend/datasources/DatasetMetadataManagerDialog.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include ImportDatasetWidget::ImportDatasetWidget(QWidget* parent) : QWidget(parent), m_categoryCompleter(new QCompleter), m_datasetCompleter(new QCompleter), m_loadingCategories(false) { const QString baseDir = QStandardPaths::standardLocations(QStandardPaths::HomeLocation).first(); QString containingDir = "labplot_data"; m_jsonDir = baseDir + QDir::separator() + containingDir + QDir::separator(); ui.setupUi(this); if(!QFile(m_jsonDir + "DatasetCategories.json").exists()) downloadCategoryFile(); loadDatasetCategoriesFromJson(); ui.lwDatasets->setSelectionMode(QAbstractItemView::SingleSelection); ui.twCategories->setSelectionMode(QAbstractItemView::SingleSelection); connect(ui.twCategories, &QTreeWidget::itemDoubleClicked, this, &ImportDatasetWidget::listDatasetsForSubcategory); connect(ui.twCategories, &QTreeWidget::itemSelectionChanged, [this] { if(!m_loadingCategories) listDatasetsForSubcategory(ui.twCategories->selectedItems().first()); }); connect(ui.leSearchDatasets, &QLineEdit::textChanged, this, &ImportDatasetWidget::scrollToDatasetListItem); connect(ui.bClearCache, &QPushButton::clicked, this, &ImportDatasetWidget::clearCache); connect(ui.leSearchCategories, &QLineEdit::textChanged, this, &ImportDatasetWidget::scrollToCategoryTreeItem); connect(ui.bRefresh, &QPushButton::clicked, this, &ImportDatasetWidget::refreshCategories); connect(ui.bNewDataset, &QPushButton::clicked, this, &ImportDatasetWidget::showDatasetMetadataManager); connect(ui.lwDatasets, &QListWidget::itemSelectionChanged, [this]() { emit datasetSelected(); }); } ImportDatasetWidget::~ImportDatasetWidget() { if(m_categoryCompleter != nullptr) delete m_categoryCompleter; if(m_datasetCompleter != nullptr) delete m_datasetCompleter; } QString ImportDatasetWidget::locateCategoryJsonFile() const { qDebug() << "Locating category file" << QStandardPaths::locate(QStandardPaths::AppDataLocation, "datasets/DatasetCategories.json"); return QStandardPaths::locate(QStandardPaths::AppDataLocation, "datasets/DatasetCategories.json"); } void ImportDatasetWidget::loadDatasetCategoriesFromJson() { qDebug() << "loading Categories"; QString filePath = m_jsonDir + "DatasetCategories.json"; QFile file(filePath); if (file.open(QIODevice::ReadOnly)) { m_loadingCategories = true; ui.lwDatasets->clear(); ui.twCategories->clear(); m_datasetsMap.clear(); QJsonDocument document = QJsonDocument::fromJson(file.readAll()); QJsonArray categoryArray = document.object().value("categories").toArray(); for(int i = 0 ; i < categoryArray.size(); ++i) { const QJsonObject currentCategory = categoryArray[i].toObject(); const QString categoryName = currentCategory.value("category_name").toString(); QTreeWidgetItem* const currentCategoryItem = new QTreeWidgetItem(QStringList(categoryName)); const QJsonArray subcategories = currentCategory.value("subcategories").toArray(); ui.twCategories->addTopLevelItem(currentCategoryItem); for(int j = 0; j < subcategories.size(); ++j) { QJsonObject currentSubCategory = subcategories[j].toObject(); QString subcategoryName = currentSubCategory.value("subcategory_name").toString(); currentCategoryItem->addChild(new QTreeWidgetItem(QStringList(subcategoryName))); const QJsonArray datasetArray = currentSubCategory.value("datasets").toArray(); // TODO remove? QVector datasets; for (const auto& dataset : datasetArray) { m_datasetsMap[categoryName][subcategoryName].push_back(dataset.toString()); } } } updateCategoryCompleter(); m_loadingCategories = false; restoreSelectedSubcategory(); file.close(); } else { qDebug("Couldn't open dataset category file"); } } void ImportDatasetWidget::restoreSelectedSubcategory() { if(m_datasetsMap.keys().contains(m_selectedCategory)) { const QTreeWidgetItem* const categoryItem = ui.twCategories->findItems(m_selectedCategory, Qt::MatchExactly).first(); if(m_datasetsMap[m_selectedCategory].keys().contains(m_selectedSubcategory)) { for(int i = 0; i < categoryItem->childCount(); ++i) { if(categoryItem->child(i)->text(0).compare(m_selectedSubcategory) == 0) { QTreeWidgetItem* const subcategoryItem = categoryItem->child(i); ui.twCategories->setCurrentItem(subcategoryItem); subcategoryItem->setSelected(true); m_selectedSubcategory.clear(); listDatasetsForSubcategory(subcategoryItem); break; } } } } } void ImportDatasetWidget::listDatasetsForSubcategory(QTreeWidgetItem* item) { if(item->childCount() == 0) { if(m_selectedSubcategory.compare(item->text(0)) != 0) { m_selectedSubcategory = item->text(0); m_selectedCategory = item->parent()->text(0); QString categoryName = item->parent()->text(0); ui.lwDatasets->clear(); for(QString dataset : m_datasetsMap[categoryName][m_selectedSubcategory]) { ui.lwDatasets->addItem(new QListWidgetItem(dataset)); } updateDatasetCompleter(); highlightLocalMetadataFiles(); } } else { if(item->text(0).compare(m_selectedCategory) != 0) { m_selectedCategory = item->text(0); m_selectedSubcategory = ""; ui.lwDatasets->clear(); item->setExpanded(true); } } } void ImportDatasetWidget::updateDatasetCompleter() { QStringList datasetList; for(int i = 0; i count(); ++i) { datasetList.append(ui.lwDatasets->item(i)->text()); } if(!datasetList.isEmpty()) { if(m_datasetCompleter != nullptr) delete m_datasetCompleter; m_datasetCompleter = new QCompleter(datasetList); m_datasetCompleter->setCompletionMode(QCompleter::PopupCompletion); m_datasetCompleter->setCaseSensitivity(Qt::CaseSensitive); ui.leSearchDatasets->setCompleter(m_datasetCompleter); } else ui.leSearchDatasets->setCompleter(nullptr); } void ImportDatasetWidget::updateCategoryCompleter() { QStringList categoryList; for (int i = 0; i < ui.twCategories->topLevelItemCount(); ++i) { categoryList.append(ui.twCategories->topLevelItem(i)->text(0)); for(int j = 0; j < ui.twCategories->topLevelItem(i)->childCount(); ++j) { categoryList.append(ui.twCategories->topLevelItem(i)->text(0) + QLatin1Char(':') + ui.twCategories->topLevelItem(i)->child(j)->text(0)); } } if(!categoryList.isEmpty()) { if(m_categoryCompleter != nullptr) delete m_categoryCompleter; m_categoryCompleter = new QCompleter(categoryList); m_categoryCompleter->setCompletionMode(QCompleter::PopupCompletion); m_categoryCompleter->setCaseSensitivity(Qt::CaseSensitive); ui.leSearchCategories->setCompleter(m_categoryCompleter); } else ui.leSearchCategories->setCompleter(nullptr); } void ImportDatasetWidget::scrollToCategoryTreeItem(const QString& rootName) { int topItemIdx = -1; for (int i = 0; i < ui.twCategories->topLevelItemCount(); ++i) if (rootName.startsWith(ui.twCategories->topLevelItem(i)->text(0))) { topItemIdx = i; break; } if (topItemIdx >= 0) { if(!rootName.contains(QLatin1Char(':'))) { ui.twCategories->scrollToItem(ui.twCategories->topLevelItem(topItemIdx), QAbstractItemView::ScrollHint::PositionAtTop); } else { int childIdx = -1; for(int j = 0; j < ui.twCategories->topLevelItem(topItemIdx)->childCount(); ++j) { if(rootName.endsWith(ui.twCategories->topLevelItem(topItemIdx)->child(j)->text(0))) { childIdx = j; break; } } if(childIdx >= 0) { ui.twCategories->scrollToItem(ui.twCategories->topLevelItem(topItemIdx)->child(childIdx), QAbstractItemView::ScrollHint::PositionAtTop); } else { ui.twCategories->scrollToItem(ui.twCategories->topLevelItem(topItemIdx), QAbstractItemView::ScrollHint::PositionAtTop); } } } } void ImportDatasetWidget::scrollToDatasetListItem(const QString& rootName) { int itemIdx = -1; for (int i = 0; i < ui.lwDatasets->count(); ++i) if (ui.lwDatasets->item(i)->text() == rootName) { itemIdx = i; break; } if (itemIdx >= 0) ui.lwDatasets->scrollToItem(ui.lwDatasets->item(itemIdx), QAbstractItemView::ScrollHint::PositionAtTop); } QString ImportDatasetWidget::getSelectedDataset() const { if (ui.lwDatasets->selectedItems().count() > 0) { return ui.lwDatasets->selectedItems().at(0)->text(); } else return QString(); } void ImportDatasetWidget::loadDatasetToProcess(DatasetHandler* datasetHandler) { const QString fileName = getSelectedDataset() + QLatin1String(".json"); downloadDatasetFile(fileName); QString filePath = m_jsonDir + m_selectedCategory + QDir::separator() + m_selectedSubcategory + QDir::separator() + fileName; if(QFile::exists(filePath)) { datasetHandler->processMetadata(filePath); } else { QMessageBox::critical(this, i18n("Can't locate file"), i18n("The metadata file for the choosen dataset can't be located")); } } -AbstractFileFilter* ImportDatasetWidget::currentFileFilter() const { - return m_currentFilter.get(); -} - void ImportDatasetWidget::showDatasetMetadataManager() { DatasetMetadataManagerDialog* dlg = new DatasetMetadataManagerDialog(this, m_datasetsMap); if (dlg->exec() == QDialog::Accepted) { const QString pathToJson = m_jsonDir + QLatin1String("DatasetCategories.json"); const QString dirPath = QFileInfo(pathToJson).dir().absolutePath(); dlg->updateDocument(pathToJson); dlg->createNewMetadata(dirPath); uploadCategoryFile(); uploadDatasetFile(dlg->getMetadataFilePath()); loadDatasetCategoriesFromJson(); } delete dlg; } void ImportDatasetWidget::downloadCategoryFile() { qDebug() << "Downloading category file"; const QString fileNameOld = QStandardPaths::locate(QStandardPaths::AppDataLocation, "datasets/DatasetCategories.json"); const QString fileNameNew =m_jsonDir + QLatin1String("DatasetCategories.json"); const QString parentDir = m_jsonDir.left(m_jsonDir.left(m_jsonDir.length() - 1).lastIndexOf(QDir::separator())); if(!QDir(m_jsonDir).exists()) { qDebug() << parentDir; QDir(parentDir).mkdir(QLatin1String("labplot_data")); } QFile::copy(fileNameOld, fileNameNew); } void ImportDatasetWidget::downloadDatasetFile(const QString& datasetName) { const QString fileNameOld = QStandardPaths::locate(QStandardPaths::AppDataLocation, QLatin1String("datasets") + QDir::separator() + datasetName); QString pathToNewFile = m_jsonDir + m_selectedCategory; if(!QDir(m_jsonDir + m_selectedCategory).exists()) { qDebug() <count(); ++i) { QListWidgetItem* const currentItem = ui.lwDatasets->item(i); const QFile file(filePath + currentItem->text() + QLatin1String(".json")); if(file.exists()) currentItem->setBackgroundColor(Qt::yellow); else currentItem->setBackgroundColor(Qt::white); } } void ImportDatasetWidget::uploadCategoryFile() { /*KNS3::UploadDialog dialog("labplot2_datasets.knsrc", this); QFile file(m_jsonDir + "DatasetCategories.json"); qDebug() << "file " << m_jsonDir + "DatasetCategories.json "<< file.exists(); qDebug() << "file can be opened: " << file.open(QIODevice::ReadOnly) << " " << file.errorString(); file.close(); QUrl payloadFile ="file:" + m_jsonDir + "DatasetCategories.json"; QFile file2(payloadFile.toLocalFile()); qDebug() << "Local file: " << payloadFile.toLocalFile(); if (!file2.open(QIODevice::ReadOnly)) { qDebug() << i18n("File not found: %1 ", payloadFile.url()); } else { qDebug() << i18n("File found: %1 ", payloadFile.url()); } file2.close(); dialog.setUploadFile("file:" + m_jsonDir + "DatasetCategories.json"); qDebug("Upload file set!"); dialog.setUploadName("Dataset Categories"); qDebug() << "Upload name set: "; dialog.exec();*/ } void ImportDatasetWidget::uploadDatasetFile(const QString& filePath) { /*KNS3::UploadDialog dialog("labplot2_datasets.knsrc", this); QFile file(filePath); qDebug() << filePath + " " << file.exists(); qDebug() << "file can be opened: " << file.open(QIODevice::ReadOnly) << " " << file.errorString(); file.close(); QUrl payloadFile ="file:" + filePath; QFile file2(payloadFile.toLocalFile()); qDebug() << "Local file: " << payloadFile.toLocalFile(); if (!file2.open(QIODevice::ReadOnly)) { qDebug() << i18n("File not found: %1 ", payloadFile.url()); } else { qDebug() << i18n("File found: %1 ", payloadFile.url()); } file2.close(); dialog.setUploadFile("file:" + filePath); qDebug("Upload file set!"); dialog.setUploadName("Dataset Categories"); qDebug() << "Upload name set: "; dialog.exec();*/ } const QMap>>& ImportDatasetWidget::getDatasetsMap() { return m_datasetsMap; } void ImportDatasetWidget::setCategory(const QString &category) { for(int i = 0; i < ui.twCategories->topLevelItemCount(); i++) { if (ui.twCategories->topLevelItem(i)->text(0).compare(category) == 0) { listDatasetsForSubcategory(ui.twCategories->topLevelItem(i)); break; } } } void ImportDatasetWidget::setSubcategory(const QString &subcategory) { for(int i = 0; i < ui.twCategories->topLevelItemCount(); i++) { if (ui.twCategories->topLevelItem(i)->text(0).compare(m_selectedCategory) == 0) { QTreeWidgetItem* categoryItem = ui.twCategories->topLevelItem(i); for(int j = 0; j childCount(); j++) { if(categoryItem->child(j)->text(0).compare(subcategory) == 0) { listDatasetsForSubcategory(categoryItem->child(j)); break; } } break; } } } void ImportDatasetWidget::setDataset(const QString &datasetName) { for(int i = 0; i < ui.lwDatasets->count() ; i++) { if(ui.lwDatasets->item(i)->text().compare(datasetName) == 0) { ui.lwDatasets->item(i)->setSelected(true); break; } } } diff --git a/src/kdefrontend/datasources/ImportDatasetWidget.h b/src/kdefrontend/datasources/ImportDatasetWidget.h index a4bc7bc0e..a251b77c5 100644 --- a/src/kdefrontend/datasources/ImportDatasetWidget.h +++ b/src/kdefrontend/datasources/ImportDatasetWidget.h @@ -1,92 +1,89 @@ /*************************************************************************** File : ImportDatasetWidget.h Project : LabPlot Description : import online dataset widget -------------------------------------------------------------------- Copyright : (C) 2019 Kovacs Ferencz (kferike98@gmail.com) ***************************************************************************/ /*************************************************************************** * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, * * Boston, MA 02110-1301 USA * * * ***************************************************************************/ #ifndef IMPORTDATASETWIDGET_H #define IMPORTDATASETWIDGET_H #include "ui_importdatasetwidget.h" #include "QWidget" -#include "backend/datasources/filters/AbstractFileFilter.h" #include "QMap" class QCompleter; class DatasetHandler; class ImportDatasetWidget : public QWidget { Q_OBJECT public: explicit ImportDatasetWidget(QWidget* parent); ~ImportDatasetWidget() override; - AbstractFileFilter* currentFileFilter() const ; QString getSelectedDataset() const; void loadDatasetToProcess(DatasetHandler* datasetHandler); QString locateCategoryJsonFile() const; const QMap>>& getDatasetsMap(); void setCategory(const QString&); void setSubcategory(const QString&); void setDataset(const QString&); private: Ui::ImportDatasetWidget ui; - mutable std::unique_ptr m_currentFilter; QMap>> m_datasetsMap; QString m_selectedSubcategory; QCompleter* m_categoryCompleter; QCompleter* m_datasetCompleter; QString m_jsonDir; bool m_loadingCategories; QString m_selectedCategory; void downloadCategoryFile(); void downloadDatasetFile(const QString&); void uploadCategoryFile(); void uploadDatasetFile(const QString&); void updateDatasetCompleter(); void updateCategoryCompleter(); void loadDatasetCategoriesFromJson(); void listDatasetsForSubcategory(QTreeWidgetItem* item); void restoreSelectedSubcategory(); void highlightLocalMetadataFiles(); private slots: void scrollToCategoryTreeItem(const QString& rootName); void scrollToDatasetListItem(const QString& rootName); void showDatasetMetadataManager(); void refreshCategories(); void clearCache(); signals: void datasetSelected(); }; #endif // IMPORTDATASETWIDGET_H