Differential D14979 Diff 40352 plugins/webarchiver/archivedialog.cpp

Changeset View

Standalone View

View Options

plugins/webarchiver/archivedialog.cpp

Show All 26 Lines
27// earlier before recursing to the to-be-removed frame, two runs are necessary to get 27​// earlier before recursing to the to-be-removed frame, two runs are necessary to get
28// a complete list of URLs that should be archived. 28​// a complete list of URLs that should be archived.
29 29​
30// Changelog 30​// Changelog
31// * replace dynamic_cast<> and ->inherits() with qobject_cast<> 31​// * replace dynamic_cast<> and ->inherits() with qobject_cast<>
32// * use QHash instead of QMap; get rid of Ordered<> class 32​// * use QHash instead of QMap; get rid of Ordered<> class
33// * fixed crash / assertion on Konqueror exit after a webpage was archived 33​// * fixed crash / assertion on Konqueror exit after a webpage was archived
34// See comment about KHTMLView parent widget in plugin_webarchiver.cpp 34​// See comment about KHTMLView parent widget in plugin_webarchiver.cpp
35// * Using KDE4/Qt4 KUrl::equals() and QUrl::fragment() to compare Urls 35​// * Using KDE4/Qt4 QUrl::equals() and QUrl::fragment() to compare Urls
dfaureUnsubmitted
Not Done

revert ;)

dfaure: revert ;)
36// * KHTML stores comment with a trailing '-'. Looks like some off-by-one bug. 36​// * KHTML stores comment with a trailing '-'. Looks like some off-by-one bug.
37// * Add mimetype indicating suffix to downloaded files. 37​// * Add mimetype indicating suffix to downloaded files.
38 38​
39// DONE CSS mentioned in <link> elements that are not parsed by Konqueror did not get their 39​// DONE CSS mentioned in <link> elements that are not parsed by Konqueror did not get their
40// href='' resolved/removed 40​// href='' resolved/removed
41 41​
42// TODO if href= etc links in a frameset refer to frames currently displayed, make links relative 42​// TODO if href= etc links in a frameset refer to frames currently displayed, make links relative
43// to archived page instead of absolute 43​// to archived page instead of absolute
Show All 26 Lines
70#include <dom/css_rule.h> 70​#include <dom/css_rule.h>
71#include <dom/css_stylesheet.h> 71​#include <dom/css_stylesheet.h>
72#include <dom/css_value.h> 72​#include <dom/css_value.h>
73 73​
74#include "archivedialog.h" 74​#include "archivedialog.h"
75#include "webarchiverdebug.h" 75​#include "webarchiverdebug.h"
76 76​
77//KDELibs4Support 77​//KDELibs4Support
78#include <kurl.h> ​
79#include <kglobal.h> 78​#include <kglobal.h>
80 79​
81// Set to true if you have a patched http-io-slave that has 80​// Set to true if you have a patched http-io-slave that has
82// improved offline-browsing functionality. 81​// improved offline-browsing functionality.
83static const bool patchedHttpSlave = false; 82​static const bool patchedHttpSlave = false;
84 83​
85#define CONTENT_TYPE "<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\" />" 84​#define CONTENT_TYPE "<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\" />"
86 85​
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Line(s)162{
171 m_widget = new ArchiveViewBase(this); 170​ m_widget = new ArchiveViewBase(this);
172 { 171​ {
173 QTreeWidgetItem *twi = m_widget->progressView->headerItem(); 172​ QTreeWidgetItem *twi = m_widget->progressView->headerItem();
174 twi->setText(0, i18n("Status")); 173​ twi->setText(0, i18n("Status"));
175 twi->setText(1, i18n("Url")); 174​ twi->setText(1, i18n("Url"));
176 } 175​ }
177 setMainWidget(m_widget); 176​ setMainWidget(m_widget);
178 177​
179 KUrl srcURL = part->url(); 178​ QUrl srcURL = part->url();
180 m_widget->urlLabel->setText(QStringLiteral("<a href=\"") + srcURL.url() + "\">" + 179​ m_widget->urlLabel->setText(QStringLiteral("<a href=\"") + srcURL.url() + "\">" +
181 KStringHandler::csqueeze(srcURL.prettyUrl(), 80) + "</a>"); 180​ KStringHandler::csqueeze(srcURL.toDisplayString(), 80) + "</a>");
182 m_widget->targetLabel->setText(QStringLiteral("<a href=\"") + filename + "\">" + 181​ m_widget->targetLabel->setText(QStringLiteral("<a href=\"") + filename + "\">" +
183 KStringHandler::csqueeze(filename, 80) + "</a>"); 182​ KStringHandler::csqueeze(filename, 80) + "</a>");
184 183​
185 //if(part->document().ownerDocument().isNull()) 184​ //if(part->document().ownerDocument().isNull())
186 // m_document = part->document(); 185​ // m_document = part->document();
187 //else 186​ //else
188 // m_document = part->document().ownerDocument(); 187​ // m_document = part->document().ownerDocument();
189 188​
Show All 20 Lines206 if (m_tarBall->open(QIODevice::WriteOnly)) {
210 209​
211 // Assign unique tarname to URLs 210​ // Assign unique tarname to URLs
212 // Split m_url2tar into Stylesheets / non stylesheets 211​ // Split m_url2tar into Stylesheets / non stylesheets
213 m_objects.clear(); 212​ m_objects.clear();
214 assert(static_cast<ssize_t>(m_url2tar.size()) - static_cast<ssize_t>(m_cssURLs.size()) >= 0); 213​ assert(static_cast<ssize_t>(m_url2tar.size()) - static_cast<ssize_t>(m_cssURLs.size()) >= 0);
215// m_objects.reserve(m_url2tar.size() - m_cssURLs.size()); 214​// m_objects.reserve(m_url2tar.size() - m_cssURLs.size());
216 215​
217 FOR_ITER(UrlTarMap, m_url2tar, u2t_it) { 216​ FOR_ITER(UrlTarMap, m_url2tar, u2t_it) {
218 const KUrl &url = u2t_it.key(); 217​ const QUrl &url = u2t_it.key();
219 DownloadInfo &info = u2t_it.value(); 218​ DownloadInfo &info = u2t_it.value();
220 219​
221 assert(info.tarName.isNull()); 220​ assert(info.tarName.isNull());
222// info.tarName = uniqTarName( url.fileName(), 0 ); 221​// info.tarName = uniqTarName( url.fileName(), 0 );
223 222​
224 // To able to append mimetype hinting suffixes to tarnames, for instance adding '.gif' to a 223​ // To able to append mimetype hinting suffixes to tarnames, for instance adding '.gif' to a
225 // webbug '87626734' adding the name to the url-to-tarname map is deferred. 224​ // webbug '87626734' adding the name to the url-to-tarname map is deferred.
226 // This cannot be done with CSS because CSS may reference each other so when URLS 225​ // This cannot be done with CSS because CSS may reference each other so when URLS
Show All 26 Lines250{
253 if (m_objects_it == m_objects.end()) { 252​ if (m_objects_it == m_objects.end()) {
254 253​
255 m_styleSheets_it = m_cssURLs.begin(); 254​ m_styleSheets_it = m_cssURLs.begin();
256 downloadStyleSheets(); 255​ downloadStyleSheets();
257 256​
258 } else { 257​ } else {
259 258​
260 m_dlurl2tar_it = (*m_objects_it); 259​ m_dlurl2tar_it = (*m_objects_it);
261 const KUrl &url = m_dlurl2tar_it.key(); 260​ const QUrl &url = m_dlurl2tar_it.key();
262 DownloadInfo &info = m_dlurl2tar_it.value(); 261​ DownloadInfo &info = m_dlurl2tar_it.value();
263 assert(m_dlurl2tar_it != m_url2tar.end()); 262​ assert(m_dlurl2tar_it != m_url2tar.end());
264 263​
265 Q_ASSERT(m_job == NULL); 264​ Q_ASSERT(m_job == NULL);
266 m_job = startDownload(url, info.part); 265​ m_job = startDownload(url, info.part);
267 connect(m_job, SIGNAL(result(KJob*)), SLOT(slotObjectFinished(KJob*))); 266​ connect(m_job, SIGNAL(result(KJob*)), SLOT(slotObjectFinished(KJob*)));
268 } 267​ }
269} 268​}
270 269​
271void ArchiveDialog::slotObjectFinished(KJob *_job) 270​void ArchiveDialog::slotObjectFinished(KJob *_job)
272{ 271​{
273 KIO::StoredTransferJob *job = qobject_cast<KIO::StoredTransferJob *>(_job); 272​ KIO::StoredTransferJob *job = qobject_cast<KIO::StoredTransferJob *>(_job);
274 Q_ASSERT(job == m_job); 273​ Q_ASSERT(job == m_job);
275 m_job = NULL; 274​ m_job = NULL;
276 const KUrl &url = m_dlurl2tar_it.key(); 275​ const QUrl &url = m_dlurl2tar_it.key();
277 DownloadInfo &info = m_dlurl2tar_it.value(); 276​ DownloadInfo &info = m_dlurl2tar_it.value();
278 277​
279 assert(info.tarName.isNull()); 278​ assert(info.tarName.isNull());
280 bool error = job->error(); 279​ bool error = job->error();
281 if (!error) { 280​ if (!error) {
282 const QString &mimetype(job->mimetype()); 281​ const QString &mimetype(job->mimetype());
283 info.tarName = uniqTarName(appendMimeTypeSuffix(url.fileName(), mimetype), 0); 282​ info.tarName = uniqTarName(appendMimeTypeSuffix(url.fileName(), mimetype), 0);
284 283​
285 QByteArray data(job->data()); 284​ QByteArray data(job->data());
286 const QString &tarName = info.tarName; 285​ const QString &tarName = info.tarName;
287 286​
288// qCDebug(WEBARCHIVERPLUGIN_LOG) << "downloaded " << url.prettyUrl() << "size=" << data.size() << "mimetype" << mimetype; 287​// qCDebug(WEBARCHIVERPLUGIN_LOG) << "downloaded " << url.toDisplayString() << "size=" << data.size() << "mimetype" << mimetype;
289 error = ! m_tarBall->writeFile(tarName, data, archivePerms, QString::null, QString::null, 288​ error = ! m_tarBall->writeFile(tarName, data, archivePerms, QString::null, QString::null,
290 m_archiveTime, m_archiveTime, m_archiveTime); 289​ m_archiveTime, m_archiveTime, m_archiveTime);
291 if (error) { 290​ if (error) {
292 qCDebug(WEBARCHIVERPLUGIN_LOG) << "Error writing to archive file"; 291​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "Error writing to archive file";
293 finishedArchiving(true); 292​ finishedArchiving(true);
294 return; 293​ return;
295 } 294​ }
296 } else { 295​ } else {
Show All 10 Lines
307{ 306​{
308 if (m_styleSheets_it == m_cssURLs.end()) { 307​ if (m_styleSheets_it == m_cssURLs.end()) {
309 308​
310 saveWebpages(); 309​ saveWebpages();
311 310​
312 } else { 311​ } else {
313 312​
314// QTimer::singleShot(3000, this, SLOT(slotDownloadStyleSheetsDelay())); 313​// QTimer::singleShot(3000, this, SLOT(slotDownloadStyleSheetsDelay()));
315 const KUrl &url = m_styleSheets_it.key(); 314​ const QUrl &url = m_styleSheets_it.key();
316 m_dlurl2tar_it = m_url2tar.find(url); 315​ m_dlurl2tar_it = m_url2tar.find(url);
317 assert(m_dlurl2tar_it != m_url2tar.end()); 316​ assert(m_dlurl2tar_it != m_url2tar.end());
318 DownloadInfo &info = m_dlurl2tar_it.value(); 317​ DownloadInfo &info = m_dlurl2tar_it.value();
319 318​
320 Q_ASSERT(m_job == NULL); 319​ Q_ASSERT(m_job == NULL);
321 m_job = startDownload(url, info.part); 320​ m_job = startDownload(url, info.part);
322 connect(m_job, SIGNAL(result(KJob*)), SLOT(slotStyleSheetFinished(KJob*))); 321​ connect(m_job, SIGNAL(result(KJob*)), SLOT(slotStyleSheetFinished(KJob*)));
323 } 322​ }
324} 323​}
325 324​
326void ArchiveDialog::slotStyleSheetFinished(KJob *_job) 325​void ArchiveDialog::slotStyleSheetFinished(KJob *_job)
327{ 326​{
328 KIO::StoredTransferJob *job = qobject_cast<KIO::StoredTransferJob *>(_job); 327​ KIO::StoredTransferJob *job = qobject_cast<KIO::StoredTransferJob *>(_job);
329 Q_ASSERT(job == m_job); 328​ Q_ASSERT(job == m_job);
330 m_job = NULL; 329​ m_job = NULL;
331 const KUrl &url = m_dlurl2tar_it.key(); 330​ const QUrl &url = m_dlurl2tar_it.key();
332 DownloadInfo &info = m_dlurl2tar_it.value(); 331​ DownloadInfo &info = m_dlurl2tar_it.value();
333 332​
334 bool error = job->error(); 333​ bool error = job->error();
335 if (! error) { 334​ if (! error) {
336 QByteArray data(job->data()); 335​ QByteArray data(job->data());
337 const QString &tarName = info.tarName; 336​ const QString &tarName = info.tarName;
338 337​
339 URLsInStyleSheet::Iterator uss_it = m_URLsInStyleSheet.find(m_styleSheets_it.value()); 338​ URLsInStyleSheet::Iterator uss_it = m_URLsInStyleSheet.find(m_styleSheets_it.value());
Show All 25 Lines362 } else {
365 qCDebug(WEBARCHIVERPLUGIN_LOG) << "download error for css url='" << url; 364​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "download error for css url='" << url;
366 } 365​ }
367 366​
368 endProgressInfo(error); 367​ endProgressInfo(error);
369 ++m_styleSheets_it; 368​ ++m_styleSheets_it;
370 downloadStyleSheets(); 369​ downloadStyleSheets();
371} 370​}
372 371​
373KIO::Job *ArchiveDialog::startDownload(const KUrl &url, KHTMLPart *part) 372​KIO::Job *ArchiveDialog::startDownload(const QUrl &url, KHTMLPart *part)
374{ 373​{
375 QTreeWidgetItem *twi = new QTreeWidgetItem; 374​ QTreeWidgetItem *twi = new QTreeWidgetItem;
376 twi->setText(0, i18n("Downloading")); 375​ twi->setText(0, i18n("Downloading"));
377 twi->setText(1, url.prettyUrl()); 376​ twi->setText(1, url.toDisplayString());
378 QTreeWidget *tw = m_widget->progressView; 377​ QTreeWidget *tw = m_widget->progressView;
379 tw->insertTopLevelItem(0, twi); 378​ tw->insertTopLevelItem(0, twi);
380 379​
381 KIO::Job *job = KIO::storedGet(url, KIO::NoReload, KIO::HideProgressInfo); 380​ KIO::Job *job = KIO::storedGet(url, KIO::NoReload, KIO::HideProgressInfo);
382 381​
383 // Use entry from cache only. Avoids re-downloading. Requires modified kio_http slave. 382​ // Use entry from cache only. Avoids re-downloading. Requires modified kio_http slave.
384 job->addMetaData(QStringLiteral("cache"), patchedHttpSlave ? "cacheonly" : "cache"); 383​ job->addMetaData(QStringLiteral("cache"), patchedHttpSlave ? "cacheonly" : "cache");
385 384​
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Line(s)449struct GetName : public GetFromPart {
453 operator QString() 452​ operator QString()
454 { 453​ {
455 return child->objectName(); 454​ return child->objectName();
456 } 455​ }
457}; 456​};
458struct GetURL : public GetFromPart { 457​struct GetURL : public GetFromPart {
459 GetURL(const KHTMLPart *child) : GetFromPart(child) { } 458​ GetURL(const KHTMLPart *child) : GetFromPart(child) { }
460 459​
461 operator KUrl() 460​ operator QUrl()
462 { 461​ {
463 return child->url(); 462​ return child->url();
464 } 463​ }
465}; 464​};
466 465​
467template< class Id2Part, class FuncObj > 466​template< class Id2Part, class FuncObj >
468static void filterFrameMappings(KHTMLPart *part, Id2Part &result) 467​static void filterFrameMappings(KHTMLPart *part, Id2Part &result)
469{ 468​{
▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Line(s)506{
529 assert(! m_framesInPart.empty()); 528​ assert(! m_framesInPart.empty());
530#if 0 529​#if 0
531 FOR_ITER(CSSURLSet, m_cssURLs, it) { 530​ FOR_ITER(CSSURLSet, m_cssURLs, it) {
532 qCDebug(WEBARCHIVERPLUGIN_LOG) << "to be downloaded stylesheet='" << it.key(); 531​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "to be downloaded stylesheet='" << it.key();
533 } 532​ }
534 FOR_ITER(URLsInStyleSheet, m_URLsInStyleSheet, ss2u_it) { 533​ FOR_ITER(URLsInStyleSheet, m_URLsInStyleSheet, ss2u_it) {
535 qCDebug(WEBARCHIVERPLUGIN_LOG) << "raw URLs in sheet='" << ss2u_it.key().href(); 534​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "raw URLs in sheet='" << ss2u_it.key().href();
536 FOR_ITER(RawHRef2FullURL, ss2u_it.data(), c2f_it) { 535​ FOR_ITER(RawHRef2FullURL, ss2u_it.data(), c2f_it) {
537 qCDebug(WEBARCHIVERPLUGIN_LOG) << " url='" << c2f_it.key() << "' -> '" << c2f_it.data().prettyUrl(); 536​ qCDebug(WEBARCHIVERPLUGIN_LOG) << " url='" << c2f_it.key() << "' -> '" << c2f_it.data().toDisplayString();
538 } 537​ }
539 } 538​ }
540 FOR_ITER(URLsInStyleElement, m_URLsInStyleElement, e2u_it) { 539​ FOR_ITER(URLsInStyleElement, m_URLsInStyleElement, e2u_it) {
541 qCDebug(WEBARCHIVERPLUGIN_LOG) << "raw URLs in style-element:"; 540​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "raw URLs in style-element:";
542 FOR_ITER(RawHRef2FullURL, e2u_it.data(), c2f_it) { 541​ FOR_ITER(RawHRef2FullURL, e2u_it.data(), c2f_it) {
543 qCDebug(WEBARCHIVERPLUGIN_LOG) << " url='" << c2f_it.key() << "' -> '" << c2f_it.data().prettyUrl(); 542​ qCDebug(WEBARCHIVERPLUGIN_LOG) << " url='" << c2f_it.key() << "' -> '" << c2f_it.data().toDisplayString();
544 } 543​ }
545 } 544​ }
546#endif 545​#endif
547} 546​}
548 547​
549void ArchiveDialog::obtainStyleSheetURLsLower(DOM::CSSStyleSheet css, RecurseData &data) 548​void ArchiveDialog::obtainStyleSheetURLsLower(DOM::CSSStyleSheet css, RecurseData &data)
550{ 549​{
551 550​
Show All 20 Lines569 case DOM::CSSRule::IMPORT_RULE: {
572 571​
573 DOM::CSSStyleSheet importSheet = cir.styleSheet(); 572​ DOM::CSSStyleSheet importSheet = cir.styleSheet();
574 if (importSheet.isNull()) { 573​ if (importSheet.isNull()) {
575 574​
576 // Given stylesheet was not downloaded / parsed by KHTML 575​ // Given stylesheet was not downloaded / parsed by KHTML
577 // Remove that URL from the stylesheet 576​ // Remove that URL from the stylesheet
578 qCDebug(WEBARCHIVERPLUGIN_LOG) << "stylesheet: invalid @import url('" << cir.href() << "')"; 577​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "stylesheet: invalid @import url('" << cir.href() << "')";
579 578​
580 raw2full.insert(cir.href().string(), KUrl()); 579​ raw2full.insert(cir.href().string(), QUrl());
581 580​
582 } else { 581​ } else {
583 582​
584 qCDebug(WEBARCHIVERPLUGIN_LOG) << "stylesheet: @import url('" << cir.href() << "') found"; 583​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "stylesheet: @import url('" << cir.href() << "') found";
585 584​
586 QString href = cir.href().string(); 585​ QString href = cir.href().string();
587 Q_ASSERT(!href.isNull()); 586​ Q_ASSERT(!href.isNull());
588 587​
589 KUrl fullURL = importSheet.baseUrl(); 588​ QUrl fullURL = importSheet.baseUrl();
590 bool inserted = insertHRefFromStyleSheet(href, raw2full, fullURL, data); 589​ bool inserted = insertHRefFromStyleSheet(href, raw2full, fullURL, data);
591 if (inserted) { 590​ if (inserted) {
592 m_cssURLs.insert(fullURL, importSheet); 591​ m_cssURLs.insert(fullURL, importSheet);
593 obtainStyleSheetURLsLower(importSheet, data); 592​ obtainStyleSheetURLsLower(importSheet, data);
594 } 593​ }
595 } 594​ }
596 } break; 595​ } break;
597 596​
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Line(s)604{
639 for (int i = 0; i != static_cast<int>(styleSheetList.length()); ++i) { 638​ for (int i = 0; i != static_cast<int>(styleSheetList.length()); ++i) {
640 DOM::StyleSheet ss = styleSheetList.item(i); 639​ DOM::StyleSheet ss = styleSheetList.item(i);
641 if (ss.isCSSStyleSheet()) { 640​ if (ss.isCSSStyleSheet()) {
642 DOM::CSSStyleSheet &css = static_cast<DOM::CSSStyleSheet &>(ss); 641​ DOM::CSSStyleSheet &css = static_cast<DOM::CSSStyleSheet &>(ss);
643 642​
644 QString href = css.href().string(); 643​ QString href = css.href().string();
645 if (! href.isNull()) { 644​ if (! href.isNull()) {
646 QString href = css.href().string(); 645​ QString href = css.href().string();
647 KUrl fullUrl = css.baseUrl(); 646​ QUrl fullUrl = css.baseUrl();
648 qCDebug(WEBARCHIVERPLUGIN_LOG) << "top-level stylesheet='" << href; 647​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "top-level stylesheet='" << href;
649 bool inserted = insertTranslateURL(fullUrl, data); 648​ bool inserted = insertTranslateURL(fullUrl, data);
650 if (inserted) { 649​ if (inserted) {
651 m_cssURLs.insert(fullUrl, css); 650​ m_cssURLs.insert(fullUrl, css);
652 } 651​ }
653 } else { 652​ } else {
654 DOM::Node node = css.ownerNode(); 653​ DOM::Node node = css.ownerNode();
655 if (! node.isNull()) { 654​ if (! node.isNull()) {
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Line(s)694 if (eurls.frameName != invalid) {
698 // This ensures the current selected frame is saved and not the default 697​ // This ensures the current selected frame is saved and not the default
699 // frame given by the original 'src' attribute 698​ // frame given by the original 'src' attribute
700 data.partFrameData->framesWithName.insert((*eurls.frameName).value, 0); 699​ data.partFrameData->framesWithName.insert((*eurls.frameName).value, 0);
701 700​
702 } else if (eurls.frameURL != invalid) { 701​ } else if (eurls.frameURL != invalid) {
703 702​
704 // URL has no 'name' attribute. This frame cannot(?) change, so 'src' should 703​ // URL has no 'name' attribute. This frame cannot(?) change, so 'src' should
705 // identify it unambigously 704​ // identify it unambigously
706 KUrl _frameURL = absoluteURL((*eurls.frameURL).value, data); 705​ QUrl _frameURL = absoluteURL((*eurls.frameURL).value, data);
707 if (!urlCheckFailed(data.part, _frameURL)) { 706​ if (!urlCheckFailed(data.part, _frameURL)) {
708 data.partFrameData->framesWithURLOnly.insert(_frameURL.url(), 0); 707​ data.partFrameData->framesWithURLOnly.insert(QUrl(_frameURL.url()), 0);
709 } 708​ }
710 709​
711 } else { 710​ } else {
712 // Ignore empty frame tags 711​ // Ignore empty frame tags
713 } 712​ }
714 713​
715 if (eurls.transURL != invalid) { 714​ if (eurls.transURL != invalid) {
716 // Kills insecure/invalid links. Frames are treated separately. 715​ // Kills insecure/invalid links. Frames are treated separately.
Show All 10 Lines725 while (! child.isNull()) {
727 obtainPartURLsLower(child, level + 1, data); 726​ obtainPartURLsLower(child, level + 1, data);
728 child = child.nextSibling(); 727​ child = child.nextSibling();
729 } 728​ }
730 } 729​ }
731} 730​}
732 731​
733// Kill insecure/invalid links. Frames are treated separately. 732​// Kill insecure/invalid links. Frames are treated separately.
734 733​
735bool ArchiveDialog::insertTranslateURL(const KUrl &fullURL, RecurseData &data) 734​bool ArchiveDialog::insertTranslateURL(const QUrl &fullURL, RecurseData &data)
736{ 735​{
737 if (!urlCheckFailed(data.part, fullURL)) { 736​ if (!urlCheckFailed(data.part, fullURL)) {
738// qCDebug(WEBARCHIVERPLUGIN_LOG) << "adding '" << fullURL << "' to to-be-downloaded URLs"; 737​// qCDebug(WEBARCHIVERPLUGIN_LOG) << "adding '" << fullURL << "' to to-be-downloaded URLs";
739 m_url2tar.insert(fullURL, DownloadInfo(QString::null, data.part)); 738​ m_url2tar.insert(fullURL, DownloadInfo(QString::null, data.part));
740 return true; 739​ return true;
741 } else { 740​ } else {
742 qCDebug(WEBARCHIVERPLUGIN_LOG) << "URL check failed on '" << fullURL << "' -- skipping"; 741​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "URL check failed on '" << fullURL << "' -- skipping";
743 return false; 742​ return false;
744 } 743​ }
745} 744​}
746 745​
747bool ArchiveDialog::insertHRefFromStyleSheet(const QString &hrefRaw, RawHRef2FullURL &raw2full, 746​bool ArchiveDialog::insertHRefFromStyleSheet(const QString &hrefRaw, RawHRef2FullURL &raw2full,
748 const KUrl &fullURL, RecurseData &data) 747​ const QUrl &fullURL, RecurseData &data)
749{ 748​{
750 bool inserted = insertTranslateURL(fullURL, data); 749​ bool inserted = insertTranslateURL(fullURL, data);
751 750​
752#if 0 751​#if 0
753 if (inserted) { 752​ if (inserted) {
754 qCDebug(WEBARCHIVERPLUGIN_LOG) << "stylesheet: found url='" 753​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "stylesheet: found url='"
755 << fullURL.prettyUrl() << "' hrefRaw='" << hrefRaw; 754​ << fullURL.toDisplayString() << "' hrefRaw='" << hrefRaw;
756 } else { 755​ } else {
757 qCDebug(WEBARCHIVERPLUGIN_LOG) << "stylesheet: killing insecure/invalid url='" 756​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "stylesheet: killing insecure/invalid url='"
758 << fullURL.prettyUrl() << "' hrefRaw='" << hrefRaw; 757​ << fullURL.toDisplayString() << "' hrefRaw='" << hrefRaw;
759 } 758​ }
760#endif 759​#endif
761 760​
762 raw2full.insert(hrefRaw, inserted ? fullURL : KUrl()); 761​ raw2full.insert(hrefRaw, inserted ? fullURL : QUrl());
763 return inserted; 762​ return inserted;
764} 763​}
765 764​
766void ArchiveDialog::parseStyleDeclaration(const KUrl &baseURL, DOM::CSSStyleDeclaration decl, 765​void ArchiveDialog::parseStyleDeclaration(const QUrl &baseURL, DOM::CSSStyleDeclaration decl,
767 RawHRef2FullURL &raw2full, RecurseData &data /*, bool verbose*/) 766​ RawHRef2FullURL &raw2full, RecurseData &data /*, bool verbose*/)
768{ 767​{
769 for (int k = 0; k != static_cast<int>(decl.length()); ++k) { 768​ for (int k = 0; k != static_cast<int>(decl.length()); ++k) {
770 DOM::DOMString item = decl.item(k); 769​ DOM::DOMString item = decl.item(k);
771 DOM::DOMString val = decl.getPropertyValue(item); 770​ DOM::DOMString val = decl.getPropertyValue(item);
772 //DOM::CSSValue csval = decl.getPropertyCSSValue(item); 771​ //DOM::CSSValue csval = decl.getPropertyCSSValue(item);
773 772​
774// qCDebug(WEBARCHIVERPLUGIN_LOG) << "style declaration " << item << ":" << val << ";"; 773​// qCDebug(WEBARCHIVERPLUGIN_LOG) << "style declaration " << item << ":" << val << ";";
775 774​
776 QString href = extractCSSURL(val.string()); 775​ QString href = extractCSSURL(val.string());
777 if (href != QString::null) { 776​ if (href != QString::null) {
778 777​
779// qCDebug(WEBARCHIVERPLUGIN_LOG) << "URL in CSS " << item << ":" << val << ";"; 778​// qCDebug(WEBARCHIVERPLUGIN_LOG) << "URL in CSS " << item << ":" << val << ";";
780 779​
781 // TODO Would like to use khtml::parseURL to remove \r, \n and similar 780​ // TODO Would like to use khtml::parseURL to remove \r, \n and similar
782 QString parsedURL = parseURL(href); 781​ QString parsedURL = parseURL(href);
783 782​
784// qCDebug(WEBARCHIVERPLUGIN_LOG) << "found URL='" << val << "' extracted='" << parsedURL << "'"; 783​// qCDebug(WEBARCHIVERPLUGIN_LOG) << "found URL='" << val << "' extracted='" << parsedURL << "'";
785 insertHRefFromStyleSheet(href, raw2full, KUrl(baseURL, parsedURL), data); 784​ insertHRefFromStyleSheet(href, raw2full, QUrl(baseURL).resolved(QUrl(parsedURL)), data);
786 } 785​ }
787 } 786​ }
788} 787​}
789 788​
790/* Saves all frames, starting from top */ 789​/* Saves all frames, starting from top */
791 790​
792bool ArchiveDialog::saveTopFrame() 791​bool ArchiveDialog::saveTopFrame()
793{ 792​{
▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Line(s)912 } else {
917 ExtractURLs eurls(nodeName, element); 916​ ExtractURLs eurls(nodeName, element);
918 917​
919 AttrList::Iterator filterOut1 = eurls.attrList.end(); 918​ AttrList::Iterator filterOut1 = eurls.attrList.end();
920 AttrList::Iterator filterOut2 = eurls.attrList.end(); 919​ AttrList::Iterator filterOut2 = eurls.attrList.end();
921 const AttrList::Iterator invalid = eurls.attrList.end(); 920​ const AttrList::Iterator invalid = eurls.attrList.end();
922 921​
923 // make URLs in hyperref links absolute 922​ // make URLs in hyperref links absolute
924 if (eurls.absURL != invalid) { 923​ if (eurls.absURL != invalid) {
925 KUrl baseurl = absoluteURL(QLatin1String(""), data); 924​ QUrl baseurl = absoluteURL(QLatin1String(""), data);
926 KUrl newurl = KUrl(baseurl, parseURL((*eurls.absURL).value)); 925​ QUrl newurl = QUrl(baseurl).resolved(QUrl(parseURL((*eurls.absURL).value)));
927 if (urlCheckFailed(data.part, newurl)) { 926​ if (urlCheckFailed(data.part, newurl)) {
928 (*eurls.absURL).value = QLatin1String(""); 927​ (*eurls.absURL).value = QLatin1String("");
929 qCDebug(WEBARCHIVERPLUGIN_LOG) << "removing invalid/insecure href='" << newurl << "'"; 928​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "removing invalid/insecure href='" << newurl << "'";
930 } else { 929​ } else {
931 // 930​ //
932 // KUrl::htmlRef() calls internally fragment()->toPercent()->toLatin1()->fromLatin1()->fromPercent() 931​ // KUrl::htmlRef() calls internally fragment()->toPercent()->toLatin1()->fromLatin1()->fromPercent()
933 // This is slow of course and there would be only a difference if there is some suburl. 932​ // This is slow of course and there would be only a difference if there is some suburl.
934 // Since we discard any urls with suburls for security reasons QUrl::fragment() is sufficient. 933​ // Since we discard any urls with suburls for security reasons QUrl::fragment() is sufficient.
935 // 934​ //
936 assert(! newurl.hasSubUrl()); // @see urlCheckFailed() 935​ assert(! hasSubUrl(newurl)); // @see urlCheckFailed()
937 if (newurl.hasFragment() && baseurl.equals(newurl, KUrl::CompareWithoutFragment)) { 936​ if (newurl.hasFragment() && baseurl.matches(newurl, QUrl::RemoveFragment)) {
dfaureUnsubmitted
Not Done

Are we sure baseurl has no fragment itself? The old code would remove it from both.

dfaure: Are we sure baseurl has no fragment itself? The old code would remove it from both.
stefanocroccoAuthorUnsubmitted
Not Done

According to the documentation for QUrl::matches, the given options are applied to both URLs before coomparing them, so there should be no problems.

stefanocrocco: According to the [documentation for `QUrl::matches`](http://doc.qt.io/qt-5/qurl.html#matches)…
dfaureUnsubmitted
Not Done

Indeed. Sorry, I read too quickly.

dfaure: Indeed. Sorry, I read too quickly.
938 (*eurls.absURL).value = QStringLiteral("#") + newurl.fragment(); 937​ (*eurls.absURL).value = QStringLiteral("#") + newurl.fragment();
939 } else { 938​ } else {
940 (*eurls.absURL).value = newurl.url(); 939​ (*eurls.absURL).value = newurl.url();
941 } 940​ }
942 } 941​ }
943 } 942​ }
944 943​
945 // make URLs of embedded objects local to tarfile 944​ // make URLs of embedded objects local to tarfile
946 if (eurls.transURL != invalid) { 945​ if (eurls.transURL != invalid) {
947 // NOTE This is a bit inefficient, because the URL is computed twice, here and when obtaining all 946​ // NOTE This is a bit inefficient, because the URL is computed twice, here and when obtaining all
948 // URLs first. However it is necessary, because two URLs that look different in the HTML frames (for 947​ // URLs first. However it is necessary, because two URLs that look different in the HTML frames (for
949 // example absolute and relative) may resolve to the same absolute URL 948​ // example absolute and relative) may resolve to the same absolute URL
950 KUrl fullURL = absoluteURL(parseURL((*eurls.transURL).value), data); 949​ QUrl fullURL = absoluteURL(parseURL((*eurls.transURL).value), data);
951 UrlTarMap::Iterator it = m_url2tar.find(fullURL); 950​ UrlTarMap::Iterator it = m_url2tar.find(fullURL);
952 if (it == m_url2tar.end()) { 951​ if (it == m_url2tar.end()) {
953 952​
954 (*eurls.transURL).value = QLatin1String(""); 953​ (*eurls.transURL).value = QLatin1String("");
955 qCDebug(WEBARCHIVERPLUGIN_LOG) << "removing invalid/insecure link='" << fullURL << "'"; 954​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "removing invalid/insecure link='" << fullURL << "'";
956 955​
957 } else { 956​ } else {
958// assert( !it.value().tarName.isNull() ); 957​// assert( !it.value().tarName.isNull() );
959 (*eurls.transURL).value = it.value().tarName; 958​ (*eurls.transURL).value = it.value().tarName;
960 } 959​ }
961 } 960​ }
962 961​
963 // Check stylesheet <link>s 962​ // Check stylesheet <link>s
964 if (eurls.cssURL != invalid) { 963​ if (eurls.cssURL != invalid) {
965 964​
966 KUrl fullURL = absoluteURL((*eurls.cssURL).value, data); 965​ QUrl fullURL = absoluteURL((*eurls.cssURL).value, data);
967 UrlTarMap::Iterator it = m_url2tar.find(fullURL); 966​ UrlTarMap::Iterator it = m_url2tar.find(fullURL);
968 967​
969 if (it == m_url2tar.end()) { 968​ if (it == m_url2tar.end()) {
970 969​
971 qCDebug(WEBARCHIVERPLUGIN_LOG) << "removing invalid/insecure CSS link='" << fullURL << "'"; 970​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "removing invalid/insecure CSS link='" << fullURL << "'";
972 (*eurls.cssURL).value = QLatin1String(""); 971​ (*eurls.cssURL).value = QLatin1String("");
973 972​
974 } else { 973​ } else {
Show All 30 Lines992 } else {
1005 1004​
1006 qCDebug(WEBARCHIVERPLUGIN_LOG) << "setting frame='" << (*eurls.frameName).value << "' to src='" 1005​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "setting frame='" << (*eurls.frameName).value << "' to src='"
1007 << (*eurls.frameURL).value; 1006​ << (*eurls.frameURL).value;
1008 } 1007​ }
1009 1008​
1010 } else if (eurls.frameURL != invalid) { 1009​ } else if (eurls.frameURL != invalid) {
1011 1010​
1012 URL2Part &u2f = data.partFrameData->framesWithURLOnly; 1011​ URL2Part &u2f = data.partFrameData->framesWithURLOnly;
1013 KUrl fullURL = absoluteURL((*eurls.frameURL).value, data); 1012​ QUrl fullURL = absoluteURL((*eurls.frameURL).value, data);
1014 URL2Part::Iterator u2f_part = u2f.find(fullURL); 1013​ URL2Part::Iterator u2f_part = u2f.find(fullURL);
1015 1014​
1016 if (u2f_part == u2f.end()) { 1015​ if (u2f_part == u2f.end()) {
1017 1016​
1018 // KHTML ignores this frame tag, so remove it here 1017​ // KHTML ignores this frame tag, so remove it here
1019 filterOut1 = eurls.frameURL; 1018​ filterOut1 = eurls.frameURL;
1020 1019​
1021 qCDebug(WEBARCHIVERPLUGIN_LOG) << "emptying frame='" << (*eurls.frameURL).value << "'"; 1020​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "emptying frame='" << (*eurls.frameURL).value << "'";
▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Line(s)1142 } else {
1144 return QString::null; 1143​ return QString::null;
1145 } 1144​ }
1146} 1145​}
1147 1146​
1148QString &ArchiveDialog::changeCSSURLs(QString &text, const RawHRef2FullURL &raw2full) 1147​QString &ArchiveDialog::changeCSSURLs(QString &text, const RawHRef2FullURL &raw2full)
1149{ 1148​{
1150 FOR_CONST_ITER(RawHRef2FullURL, raw2full, r2f_it) { 1149​ FOR_CONST_ITER(RawHRef2FullURL, raw2full, r2f_it) {
1151 const QString &raw = r2f_it.key(); 1150​ const QString &raw = r2f_it.key();
1152 const KUrl &fullURL = r2f_it.value(); 1151​ const QUrl &fullURL = r2f_it.value();
1153 if (fullURL.isValid()) { 1152​ if (fullURL.isValid()) {
1154 UrlTarMap::Iterator utm_it = m_url2tar.find(fullURL); 1153​ UrlTarMap::Iterator utm_it = m_url2tar.find(fullURL);
1155 if (utm_it != m_url2tar.end()) { 1154​ if (utm_it != m_url2tar.end()) {
1156 const QString &tarName = utm_it.value().tarName; 1155​ const QString &tarName = utm_it.value().tarName;
1157// assert(! tarName.isNull()); 1156​// assert(! tarName.isNull());
1158 1157​
1159 qCDebug(WEBARCHIVERPLUGIN_LOG) << "changeCSSURLs: url=" << raw << " -> " << tarName; 1158​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "changeCSSURLs: url=" << raw << " -> " << tarName;
1160 text.replace(raw, tarName); 1159​ text.replace(raw, tarName);
▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Line(s)1270{
1272 FOR_ITER(AttrList, attrList, it) { 1271​ FOR_ITER(AttrList, attrList, it) {
1273 if ((*it).name == attr) { 1272​ if ((*it).name == attr) {
1274 return it; 1273​ return it;
1275 } 1274​ }
1276 } 1275​ }
1277 return attrList.end(); 1276​ return attrList.end();
1278} 1277​}
1279 1278​
1280KUrl ArchiveDialog::absoluteURL(const QString &partURL, RecurseData &data) 1279​QUrl ArchiveDialog::absoluteURL(const QString &partURL, RecurseData &data)
1281{ 1280​{
1282 if (data.baseSeen) { 1281​ if (data.baseSeen) {
1283 return KUrl(data.document.completeURL(partURL).string()); 1282​ return QUrl(data.document.completeURL(partURL).string());
1284 } else { 1283​ } else {
1285 return KUrl(data.part->url(), partURL); 1284​ return QUrl(data.part->url()).resolved(QUrl(partURL));
1286 } 1285​ }
1287} 1286​}
1288 1287​
1289// TODO Should be khtml::parseURL 1288​// TODO Should be khtml::parseURL
1290QString ArchiveDialog::parseURL(const QString &rawurl) 1289​QString ArchiveDialog::parseURL(const QString &rawurl)
1291{ 1290​{
1292 QString result = rawurl; 1291​ QString result = rawurl;
1293 return result.replace(QRegExp("[\\x0000-\\x000D]"), QLatin1String("")); 1292​ return result.replace(QRegExp("[\\x0000-\\x000D]"), QLatin1String(""));
1294} 1293​}
1295 1294​
1296QString ArchiveDialog::uniqTarName(const QString &suggestion, KHTMLPart *part) 1295​QString ArchiveDialog::uniqTarName(const QString &suggestion, KHTMLPart *part)
1297{ 1296​{
1298 1297​
1299 QString result = suggestion; 1298​ QString result = suggestion;
1300 1299​
1301 // Name clash -> add unique id 1300​ // Name clash -> add unique id
1302 while (result.isEmpty() || m_tarName2part.contains(result)) { 1301​ while (result.isEmpty() || m_tarName2part.contains(result)) {
1303 result = QString::number(m_uniqId++) + suggestion; 1302​ result = QString::number(m_uniqId++) + suggestion;
1304 } 1303​ }
1305 m_tarName2part.insert(result, part); 1304​ m_tarName2part.insert(result, part);
1306 1305​
1307 return result; 1306​ return result;
1308} 1307​}
1309 1308​
1310bool ArchiveDialog::urlCheckFailed(KHTMLPart *part, const KUrl &fullURL) 1309​bool ArchiveDialog::urlCheckFailed(KHTMLPart *part, const QUrl &fullURL)
1311{ 1310​{
1312 if (!fullURL.isValid()) { 1311​ if (!fullURL.isValid()) {
1313 return true; 1312​ return true;
1314 } 1313​ }
1315// kDebug() << fullURL.prettyUrl() << " hasSubURL()=" << fullURL.hasSubUrl(); ​
1316 if (fullURL.hasSubUrl()) { ​
1317 return true; ​
1318 } ​
1319 1314​
1320 QString prot = fullURL.protocol(); 1315​ QString prot = fullURL.scheme();
1321 bool protFile = (prot == QLatin1String("file")); 1316​ bool protFile = (prot == QLatin1String("file"));
1322 if (part->onlyLocalReferences() && !protFile) { 1317​ if (part->onlyLocalReferences() && !protFile) {
1323 return true; 1318​ return true;
1324 } 1319​ }
1325 1320​
1326 bool protHttp = (prot == QLatin1String("http")) || (prot == QLatin1String("https")); 1321​ bool protHttp = (prot == QLatin1String("http")) || (prot == QLatin1String("https"));
1327 if (!protFile && !protHttp) { 1322​ if (!protFile && !protHttp) {
1328 return true; 1323​ return true;
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Line(s)1365 if (! patterns.isEmpty()) {
1372 suffix.replace('*', QString::null); 1367​ suffix.replace('*', QString::null);
1373 filename += suffix; 1368​ filename += suffix;
1374 qCDebug(WEBARCHIVERPLUGIN_LOG) << "appended missing mimetype suffix, returning" << filename; 1369​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "appended missing mimetype suffix, returning" << filename;
1375 } else { 1370​ } else {
1376 qCDebug(WEBARCHIVERPLUGIN_LOG) << "mimetype" << mimetype << " has no pattern list, this is bad"; 1371​ qCDebug(WEBARCHIVERPLUGIN_LOG) << "mimetype" << mimetype << " has no pattern list, this is bad";
1377 Q_ASSERT(0); 1372​ Q_ASSERT(0);
1378 } 1373​ }
1379 return filename; 1374​ return filename;
1380} 1375​}
dfaureUnsubmitted
Not Done

suburls are not supported anymore, they were a KDE3 thing.

OK except for error:/, in a way, that's still used.
But kioslaves can't be chained with suburls.

dfaure: suburls are not supported anymore, they were a KDE3 thing. OK except for error:/, in a way…
1381 ​