diff --git a/language/backgroundparser/tests/test_backgroundparser.cpp b/language/backgroundparser/tests/test_backgroundparser.cpp index 0075ce4c63..6548d411af 100644 --- a/language/backgroundparser/tests/test_backgroundparser.cpp +++ b/language/backgroundparser/tests/test_backgroundparser.cpp @@ -1,328 +1,390 @@ /* * This file is part of KDevelop * * Copyright 2012 by Sven Brauch * Copyright 2012 by Milian Wolff * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU Library General Public License as * published by the Free Software Foundation; either version 2 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "test_backgroundparser.h" #include #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include #include "testlanguagesupport.h" #include "testparsejob.h" QTEST_MAIN(TestBackgroundparser) #define QVERIFY_RETURN(statement, retval) \ do { if (!QTest::qVerify((statement), #statement, "", __FILE__, __LINE__)) return retval; } while (0) using namespace KDevelop; JobPlan::JobPlan() { } void JobPlan::addJob(const JobPrototype& job) { m_jobs << job; } void JobPlan::clear() { m_jobs.clear(); m_finishedJobs.clear(); m_createdJobs.clear(); } void JobPlan::parseJobCreated(ParseJob* job) { // e.g. for the benchmark if (m_jobs.isEmpty()) { return; } TestParseJob* testJob = dynamic_cast(job); Q_ASSERT(testJob); qDebug() << "assigning propierties for created job" << testJob->document().toUrl(); testJob->duration_ms = jobForUrl(testJob->document()).m_duration; m_createdJobs.append(testJob->document()); } bool JobPlan::runJobs(int timeoutMS) { // add parse jobs foreach(const JobPrototype& job, m_jobs) { ICore::self()->languageController()->backgroundParser()->addDocument( job.m_url, TopDUContext::Empty, job.m_priority, this, job.m_flags ); } ICore::self()->languageController()->backgroundParser()->parseDocuments(); QElapsedTimer t; t.start(); while ( !t.hasExpired(timeoutMS) && m_jobs.size() != m_finishedJobs.size() ) { QTest::qWait(50); } QVERIFY_RETURN(m_jobs.size() == m_createdJobs.size(), false); QVERIFY_RETURN(m_finishedJobs.size() == m_jobs.size(), false); // verify they're started in the right order int currentBestPriority = BackgroundParser::BestPriority; foreach ( const IndexedString& url, m_createdJobs ) { const JobPrototype p = jobForUrl(url); QVERIFY_RETURN(p.m_priority >= currentBestPriority, false); currentBestPriority = p.m_priority; } return true; } JobPrototype JobPlan::jobForUrl(const IndexedString& url) { foreach(const JobPrototype& job, m_jobs) { if (job.m_url == url) { return job; } } return JobPrototype(); } void JobPlan::updateReady(const IndexedString& url, const ReferencedTopDUContext& /*context*/) { qDebug() << "update ready on " << url.toUrl(); const JobPrototype job = jobForUrl(url); QVERIFY(job.m_url.toUrl().isValid()); if (job.m_flags & ParseJob::RequiresSequentialProcessing) { // ensure that all jobs that respect sequential processing // with lower priority have been run foreach(const JobPrototype& otherJob, m_jobs) { if (otherJob.m_url == job.m_url) { continue; } if (otherJob.m_flags & ParseJob::RespectsSequentialProcessing && otherJob.m_priority < job.m_priority) { QVERIFY(m_finishedJobs.contains(otherJob.m_url)); } } } QVERIFY(!m_finishedJobs.contains(job.m_url)); m_finishedJobs << job.m_url; } void TestBackgroundparser::initTestCase() { AutoTestShell::init(); TestCore* core = TestCore::initialize(Core::NoUi); DUChain::self()->disablePersistentStorage(); TestLanguageController* langController = new TestLanguageController(core); core->setLanguageController(langController); langController->backgroundParser()->setThreadCount(4); + langController->backgroundParser()->abortAllJobs(); - auto testLang = new TestLanguageSupport(this); - connect(testLang, &TestLanguageSupport::parseJobCreated, + m_langSupport = new TestLanguageSupport(this); + connect(m_langSupport, &TestLanguageSupport::parseJobCreated, &m_jobPlan, &JobPlan::parseJobCreated); - langController->addTestLanguage(testLang, QStringList() << QStringLiteral("text/plain")); + langController->addTestLanguage(m_langSupport, QStringList() << QStringLiteral("text/plain")); const auto languages = langController->languagesForUrl(QUrl::fromLocalFile(QStringLiteral("/foo.txt"))); QCOMPARE(languages.size(), 1); - QCOMPARE(languages.first(), testLang); + QCOMPARE(languages.first(), m_langSupport); } void TestBackgroundparser::cleanupTestCase() { TestCore::shutdown(); + m_langSupport = nullptr; } void TestBackgroundparser::init() { m_jobPlan.clear(); } void TestBackgroundparser::testShutdownWithRunningJobs() { m_jobPlan.clear(); // prove that background parsing happens with sequential flags although there is a high-priority // foreground thread (active document being edited, ...) running all the time. // the long-running high-prio job m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile(QStringLiteral("/test_fgt_hp.txt")), -500, ParseJob::IgnoresSequentialProcessing, 1000)); // add parse jobs foreach(const JobPrototype& job, m_jobPlan.m_jobs) { ICore::self()->languageController()->backgroundParser()->addDocument( job.m_url, TopDUContext::Empty, job.m_priority, this, job.m_flags ); } ICore::self()->languageController()->backgroundParser()->parseDocuments(); QTest::qWait(50); // shut down with running jobs, make sure we don't crash cleanupTestCase(); // restart again to restore invariant (core always running in test functions) initTestCase(); } void TestBackgroundparser::testParseOrdering_foregroundThread() { m_jobPlan.clear(); // prove that background parsing happens with sequential flags although there is a high-priority // foreground thread (active document being edited, ...) running all the time. // the long-running high-prio job m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile(QStringLiteral("/test_fgt_hp.txt")), -500, ParseJob::IgnoresSequentialProcessing, 630)); // several small background jobs for ( int i = 0; i < 10; i++ ) { m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile("/test_fgt_lp__" + QString::number(i) + ".txt"), i, ParseJob::FullSequentialProcessing, 40)); } // not enough time if the small jobs run after the large one QVERIFY(m_jobPlan.runJobs(700)); } void TestBackgroundparser::testParseOrdering_noSequentialProcessing() { m_jobPlan.clear(); for ( int i = 0; i < 20; i++ ) { // create jobs with no sequential processing, and different priorities m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile("/test_nsp1__" + QString::number(i) + ".txt"), i, ParseJob::IgnoresSequentialProcessing, i)); } for ( int i = 0; i < 8; i++ ) { // create a few more jobs with the same priority m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile("/test_nsp2__" + QString::number(i) + ".txt"), 10, ParseJob::IgnoresSequentialProcessing, i)); } QVERIFY(m_jobPlan.runJobs(1000)); } void TestBackgroundparser::testParseOrdering_lockup() { m_jobPlan.clear(); for ( int i = 3; i > 0; i-- ) { // add 3 jobs which do not care about sequential processing, at 4 threads it should take no more than 1s to process them m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile("/test" + QString::number(i) + ".txt"), i, ParseJob::IgnoresSequentialProcessing, 200)); } // add one job which requires sequential processing with high priority m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile(QStringLiteral("/test_hp.txt")), -200, ParseJob::FullSequentialProcessing, 200)); // verify that the low-priority nonsequential jobs are run simultaneously with the other one. QVERIFY(m_jobPlan.runJobs(700)); } void TestBackgroundparser::testParseOrdering_simple() { m_jobPlan.clear(); for ( int i = 20; i > 0; i-- ) { // the job with priority i should be at place i in the finished list // (lower priority value -> should be parsed first) ParseJob::SequentialProcessingFlags flags = ParseJob::FullSequentialProcessing; m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile("/test" + QString::number(i) + ".txt"), i, flags)); } // also add a few jobs which ignore the processing for ( int i = 0; i < 5; ++i ) { m_jobPlan.addJob(JobPrototype(QUrl::fromLocalFile("/test2-" + QString::number(i) + ".txt"), BackgroundParser::NormalPriority, ParseJob::IgnoresSequentialProcessing)); } QVERIFY(m_jobPlan.runJobs(1000)); } void TestBackgroundparser::benchmark() { const int jobs = 10000; QVector jobUrls; jobUrls.reserve(jobs); for ( int i = 0; i < jobs; ++i ) { jobUrls << IndexedString("/test" + QString::number(i) + ".txt"); } QBENCHMARK { foreach ( const IndexedString& url, jobUrls ) { ICore::self()->languageController()->backgroundParser()->addDocument(url); } ICore::self()->languageController()->backgroundParser()->parseDocuments(); while ( ICore::self()->languageController()->backgroundParser()->queuedCount() ) { QTest::qWait(50); } } } void TestBackgroundparser::benchmarkDocumentChanges() { KTextEditor::Editor* editor = KTextEditor::Editor::instance(); QVERIFY(editor); KTextEditor::Document* doc = editor->createDocument(this); QVERIFY(doc); QTemporaryFile file; QVERIFY(file.open()); doc->saveAs(QUrl::fromLocalFile(file.fileName())); DocumentChangeTracker tracker(doc); doc->setText(QStringLiteral("hello world")); // required for proper benchmark results doc->createView(0); QBENCHMARK { for ( int i = 0; i < 5000; i++ ) { { KTextEditor::Document::EditingTransaction t(doc); doc->insertText(KTextEditor::Cursor(0, 0), QStringLiteral("This is a test line.\n")); } QApplication::processEvents(); } } doc->clear(); doc->save(); } +// see also: http://bugs.kde.org/355100 +void TestBackgroundparser::testNoDeadlockInJobCreation() +{ + if (!qEnvironmentVariableIntValue("TEST_BUG_355100")) { + QSKIP("Skipping deadlock to keep CI working. Set TEST_BUG_355100=1 env var to run it yourself."); + } + + m_jobPlan.clear(); + + // we need to run the background thread first (best priority) + const auto runUrl = QUrl::fromLocalFile(QStringLiteral("/lockInRun.txt")); + const auto run = IndexedString(runUrl); + m_jobPlan.addJob(JobPrototype(runUrl, BackgroundParser::BestPriority, + ParseJob::IgnoresSequentialProcessing, 0)); + + // before handling the foreground code (worst priority) + const auto ctorUrl = QUrl::fromLocalFile(QStringLiteral("/lockInCtor.txt")); + const auto ctor = IndexedString(ctorUrl); + m_jobPlan.addJob(JobPrototype(ctorUrl, BackgroundParser::WorstPriority, + ParseJob::IgnoresSequentialProcessing, 0)); + + // make sure that the background thread has the duchain locked for write + QSemaphore semaphoreA; + // make sure the foreground thread is inside the parse job ctor + QSemaphore semaphoreB; + + // actually distribute the complicate code across threads to trigger the + // deadlock reliably + QObject::connect(m_langSupport, &TestLanguageSupport::aboutToCreateParseJob, + m_langSupport, [&] (const IndexedString& url, ParseJob** job) { + if (url == run) { + auto testJob = new TestParseJob(url, m_langSupport); + testJob->run_callback = [&] () { + // this is run in the background parse thread + DUChainWriteLocker lock; + semaphoreA.release(); + // sync with the foreground parse job ctor + semaphoreB.acquire(); + // this is acquiring the background parse lock + // we want to support this order - i.e. DUChain -> Background Parser + ICore::self()->languageController()->backgroundParser()->isQueued(url); + }; + *job = testJob; + } else if (url == ctor) { + // this is run in the foreground, essentially the same + // as code run within the parse job ctor + semaphoreA.acquire(); + semaphoreB.release(); + // Note how currently, the background parser is locked while creating a parse job + // thus locking the duchain here used to trigger a lock order inversion + DUChainReadLocker lock; + *job = new TestParseJob(url, m_langSupport); + } + }, Qt::DirectConnection); + + // should be able to run quickly, if no deadlock occurs + QVERIFY(m_jobPlan.runJobs(500)); +} diff --git a/language/backgroundparser/tests/test_backgroundparser.h b/language/backgroundparser/tests/test_backgroundparser.h index 2ce0ea4e42..c27f570060 100644 --- a/language/backgroundparser/tests/test_backgroundparser.h +++ b/language/backgroundparser/tests/test_backgroundparser.h @@ -1,113 +1,116 @@ /* * This file is part of KDevelop * * Copyright 2012 by Sven Brauch * Copyright 2012 by Milian Wolff * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU Library General Public License as * published by the Free Software Foundation; either version 2 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef KDEVPLATFORM_TEST_BACKGROUNDPARSER_H #define KDEVPLATFORM_TEST_BACKGROUNDPARSER_H #include #include #include #include #include "testlanguagesupport.h" class JobPrototype { public: JobPrototype() : m_priority(0) , m_duration(0) , m_flags(ParseJob::IgnoresSequentialProcessing) { } JobPrototype(const QUrl& url, int priority, ParseJob::SequentialProcessingFlags flags, int duration = 0) : m_url(url) , m_priority(priority) , m_duration(duration) , m_flags(flags) { Q_ASSERT(url.isValid()); } IndexedString m_url; int m_priority; int m_duration; ParseJob::SequentialProcessingFlags m_flags; }; Q_DECLARE_TYPEINFO(JobPrototype, Q_MOVABLE_TYPE); class TestParseJob; class JobPlan : public QObject { Q_OBJECT public: JobPlan(); void addJob(const JobPrototype& job); bool runJobs(int timeoutMS); void clear(); JobPrototype jobForUrl(const IndexedString& url); private slots: void updateReady(const KDevelop::IndexedString& url, const KDevelop::ReferencedTopDUContext& context); void parseJobCreated(KDevelop::ParseJob*); private: friend class TestBackgroundparser; QVector m_jobs; QVector m_finishedJobs; QVector m_createdJobs; }; class TestBackgroundparser : public QObject { Q_OBJECT private slots: void initTestCase(); void cleanupTestCase(); void init(); void testShutdownWithRunningJobs(); void testParseOrdering_simple(); void testParseOrdering_lockup(); void testParseOrdering_foregroundThread(); void testParseOrdering_noSequentialProcessing(); + void testNoDeadlockInJobCreation(); + void benchmark(); void benchmarkDocumentChanges(); private: JobPlan m_jobPlan; + TestLanguageSupport *m_langSupport = nullptr; }; #endif // KDEVPLATFORM_TEST_BACKGROUNDPARSER_H diff --git a/language/backgroundparser/tests/testlanguagesupport.cpp b/language/backgroundparser/tests/testlanguagesupport.cpp index 82426ce840..4f59cc6d82 100644 --- a/language/backgroundparser/tests/testlanguagesupport.cpp +++ b/language/backgroundparser/tests/testlanguagesupport.cpp @@ -1,42 +1,46 @@ /* * This file is part of KDevelop * * Copyright 2012 by Sven Brauch * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU Library General Public License as * published by the Free Software Foundation; either version 2 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "testlanguagesupport.h" #include "testparsejob.h" #include "test_backgroundparser.h" #include using namespace KDevelop; ParseJob* TestLanguageSupport::createParseJob(const IndexedString& url) { qDebug() << "creating test language support parse job"; - TestParseJob* job = new TestParseJob(url, this); + ParseJob* job = nullptr; + emit aboutToCreateParseJob(url, &job); + if (!job) { + job = new TestParseJob(url, this); + } emit parseJobCreated(job); return job; } QString TestLanguageSupport::name() const { return QStringLiteral("TestLanguageSupport"); } diff --git a/language/backgroundparser/tests/testlanguagesupport.h b/language/backgroundparser/tests/testlanguagesupport.h index bb85bf00b3..1b5fad57b7 100644 --- a/language/backgroundparser/tests/testlanguagesupport.h +++ b/language/backgroundparser/tests/testlanguagesupport.h @@ -1,45 +1,46 @@ /* * This file is part of KDevelop * * Copyright 2012 by Sven Brauch * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU Library General Public License as * published by the Free Software Foundation; either version 2 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef KDEVPLATFORM_TESTLANGUAGESUPPORT_H #define KDEVPLATFORM_TESTLANGUAGESUPPORT_H #include "language/interfaces/ilanguagesupport.h" #include using namespace KDevelop; class TestLanguageSupport : public QObject, public KDevelop::ILanguageSupport { Q_OBJECT Q_INTERFACES(KDevelop::ILanguageSupport) public: using QObject::QObject; KDevelop::ParseJob* createParseJob(const IndexedString& url) override; QString name() const override; signals: + void aboutToCreateParseJob(const IndexedString& url, KDevelop::ParseJob** job); void parseJobCreated(KDevelop::ParseJob* job); }; #endif diff --git a/language/backgroundparser/tests/testparsejob.cpp b/language/backgroundparser/tests/testparsejob.cpp index 2a064c7921..59977d1150 100644 --- a/language/backgroundparser/tests/testparsejob.cpp +++ b/language/backgroundparser/tests/testparsejob.cpp @@ -1,51 +1,55 @@ /* * This file is part of KDevelop * * Copyright 2012 by Sven Brauch * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU Library General Public License as * published by the Free Software Foundation; either version 2 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "testparsejob.h" + #include TestParseJob::TestParseJob(const IndexedString& url, ILanguageSupport* languageSupport) : ParseJob(url, languageSupport) , duration_ms(0) { } void TestParseJob::run(ThreadWeaver::JobPointer, ThreadWeaver::Thread*) { - qDebug() << "Running parse job for" << document().toUrl(); + qDebug() << "Running parse job for" << document(); + if (run_callback) { + run_callback(); + } if (duration_ms) { qDebug() << "waiting" << duration_ms << "ms"; QTest::qWait(duration_ms); } } ControlFlowGraph* TestParseJob::controlFlowGraph() { return 0; } DataAccessRepository* TestParseJob::dataAccessInformation() { return 0; } diff --git a/language/backgroundparser/tests/testparsejob.h b/language/backgroundparser/tests/testparsejob.h index f2e75d220d..6c377f924f 100644 --- a/language/backgroundparser/tests/testparsejob.h +++ b/language/backgroundparser/tests/testparsejob.h @@ -1,41 +1,44 @@ /* * This file is part of KDevelop * * Copyright 2012 by Sven Brauch * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU Library General Public License as * published by the Free Software Foundation; either version 2 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef KDEVPLATFORM_TESTPARSEJOB_H #define KDEVPLATFORM_TESTPARSEJOB_H #include "language/backgroundparser/parsejob.h" +#include + using namespace KDevelop; class TestParseJob : public KDevelop::ParseJob { Q_OBJECT public: TestParseJob(const IndexedString& url, ILanguageSupport* languageSupport); void run(ThreadWeaver::JobPointer self, ThreadWeaver::Thread* thread) override; ControlFlowGraph* controlFlowGraph() override; DataAccessRepository* dataAccessInformation() override; int duration_ms; + std::function run_callback; }; #endif