Differential D10621 Diff 30448 src/indexer/katehighlightingindexer.cpp

Changeset View

Standalone View

View Options

src/indexer/katehighlightingindexer.cpp

Show All 26 Lines
27#include <QRegularExpression> 27​#include <QRegularExpression>
28#include <QDebug> 28​#include <QDebug>
29 29​
30#ifdef QT_XMLPATTERNS_LIB 30​#ifdef QT_XMLPATTERNS_LIB
31#include <QXmlSchema> 31​#include <QXmlSchema>
32#include <QXmlSchemaValidator> 32​#include <QXmlSchemaValidator>
33#endif 33​#endif
34 34​
35​#include <algorithm>
36​#include <iterator>
37​#include <type_traits>
38​
35namespace { 39​namespace {
36 40​
41​namespace Xml {
42​
43​bool attrToBool(const QStringRef &str)
44​{
45​ return str == QLatin1String("1") || str.compare(QLatin1String("true"), Qt::
46​CaseInsensitive) == 0;
47​}
48​
49​}
50​
37QStringList readListing(const QString &fileName) 51​QStringList readListing(const QString &fileName)
38{ 52​{
39 QFile file(fileName); 53​ QFile file(fileName);
40 if (!file.open(QIODevice::ReadOnly)) { 54​ if (!file.open(QIODevice::ReadOnly)) {
41 return QStringList(); 55​ return QStringList();
42 } 56​ }
43 57​
44 QXmlStreamReader xml(&file); 58​ QXmlStreamReader xml(&file);
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Line(s)
112{ 126​{
113 const bool testChar1 = xml.name() == QLatin1String("Detect2Chars"); 127​ const bool testChar1 = xml.name() == QLatin1String("Detect2Chars");
114 const bool testChar = testChar1 || xml.name() == QLatin1String("DetectChar"); 128​ const bool testChar = testChar1 || xml.name() == QLatin1String("DetectChar");
115 129​
116 if (testChar) { 130​ if (testChar) {
117 const QString c = xml.attributes().value(QLatin1String("char")).toString(); 131​ const QString c = xml.attributes().value(QLatin1String("char")).toString();
118 if (c.size() != 1) { 132​ if (c.size() != 1) {
119 qWarning() << hlFilename << "line" << xml.lineNumber() << "'char' must contain exactly one char:" << c; 133​ qWarning() << hlFilename << "line" << xml.lineNumber() << "'char' must contain exactly one char:" << c;
134​ return false;
120 } 135​ }
121 } 136​ }
122 137​
123 if (testChar1) { 138​ if (testChar1) {
124 const QString c = xml.attributes().value(QLatin1String("char1")).toString(); 139​ const QString c = xml.attributes().value(QLatin1String("char1")).toString();
125 if (c.size() != 1) { 140​ if (c.size() != 1) {
126 qWarning() << hlFilename << "line" << xml.lineNumber() << "'char1' must contain exactly one char:" << c; 141​ qWarning() << hlFilename << "line" << xml.lineNumber() << "'char1' must contain exactly one char:" << c;
142​ return false;
127 } 143​ }
128 } 144​ }
129 145​
130 return true; 146​ return true;
131} 147​}
132 148​
133//! Search for rules with lookAhead="true" and context="#stay". 149​//! Search for rules with lookAhead="true" and context="#stay".
134//! This would cause an infinite loop. 150​//! This would cause an infinite loop.
135bool checkLookAhead(const QString &hlFilename, QXmlStreamReader &xml) 151​bool checkLookAhead(const QString &hlFilename, QXmlStreamReader &xml)
136{ 152​{
137 if (xml.attributes().hasAttribute(QStringLiteral("lookAhead"))) { 153​ if (xml.attributes().hasAttribute(QStringLiteral("lookAhead"))) {
138 auto lookAhead = xml.attributes().value(QStringLiteral("lookAhead")); 154​ auto lookAhead = xml.attributes().value(QStringLiteral("lookAhead"));
139 if (lookAhead == QStringLiteral("true")) { 155​ if (Xml::attrToBool(lookAhead)) {
140 auto context = xml.attributes().value(QStringLiteral("context")); 156​ auto context = xml.attributes().value(QStringLiteral("context"));
141 if (context == QStringLiteral("#stay")) { 157​ if (context == QStringLiteral("#stay")) {
142 qWarning() << hlFilename << "line" << xml.lineNumber() << "Infinite loop: lookAhead with context #stay"; 158​ qWarning() << hlFilename << "line" << xml.lineNumber() << "Infinite loop: lookAhead with context #stay";
143 return false; 159​ return false;
144 } 160​ }
145 } 161​ }
146 } 162​ }
147 return true; 163​ return true;
▲ Show 20 Lines • Show All 222 Lines • ▼ Show 20 Line(s)345public:
370 } 386​ }
371 387​
372private: 388​private:
373 QString m_filename; 389​ QString m_filename;
374 QSet<QString> m_usedAttributeNames; 390​ QSet<QString> m_usedAttributeNames;
375 QSet<QString> m_existingAttributeNames; 391​ QSet<QString> m_existingAttributeNames;
376}; 392​};
377 393​
394​//! Proposes to replace StringDetect with DetectChar or Detect2Chars
395​bool suggestForStringDetect(const QString &hlFilename, QXmlStreamReader &xml)
396​{
397​ if (xml.name() == QLatin1String("StringDetect") && !Xml::attrToBool(xml.attributes().value(QLatin1String("dynamic")))) {
398​ if (!Xml::attrToBool(xml.attributes().value(QLatin1String("insensitive")))) {
399​ const auto str = xml.attributes().value(QLatin1String("String")).toString();
400​ if (str.size() == 1) {
401​ qWarning() << hlFilename << "line" << xml.lineNumber() << "StringDetect candidate for DetectChar:" << str;
402​ return false;
403​ }
404​ else if (str.size() == 2) {
405​ qWarning() << hlFilename << "line" << xml.lineNumber() << "StringDetect candidate for Detect2Chars:" << str;
406​ return false;
407​ }
408​ }
409​ }
410​
411​ return true;
412​}
413​
414​//! Proposes to replace AnyChar with DetectChar
415​bool suggestForAnyChar(const QString &hlFilename, QXmlStreamReader &xml)
416​{
417​ if (xml.name() == QLatin1String("AnyChar")) {
418​ const auto string = xml.attributes().value(QLatin1String("String")).toString();
419​ if (string.size() == 1) {
420​ qWarning() << hlFilename << "line" << xml.lineNumber() << "AnyChar candidate for DetectChar:" << string;
421​ return false;
422​ }
423​ }
424​
425​ return true;
426​}
427​
428​struct SuggestPattern
429​{
430​ enum Flags
431​ {
432​ NoFlag,
433​ HasInsensitiveAttribute = 1,
434​ HasDynamicAttribute = 2,
435​ OnlyIfColumn0 = 4,
436​ };
437​
438​ QLatin1String suggest;
439​ QRegularExpression regex;
440​ Flags flags;
441​
442​ SuggestPattern(const char *suggest, const QString &stringRegex, Flags flags)
443​ : suggest(suggest)
444​ , regex(stringRegex)
445​ , flags(flags)
446​ {
447​ Q_ASSERT(regex.isValid());
448​ }
449​};
450​
451​SuggestPattern::Flags operator|(const SuggestPattern::Flags &a, const SuggestPattern::Flags &b)
452​{
453​ return static_cast<SuggestPattern::Flags>(int(a) | int(b));
454​}
455​
456​#define EscapedChar R"(\\[0-9]{2,3}|\\x[0-9a-fA-F]{2}|\\[^bBwWsSdD])"
457​#define SingleChar "(?:" \
458​ EscapedChar \
459​"|" "\\[(?:[^^\\\\]|" EscapedChar ")\\]" \
460​"|" "[^^\\\\$.+*?|{[]" \
461​")"
462​// ( `C` | `(C)` | `(?:C)` ) (`{1}` | `{1,1}`)?
463​#define CharReg "(?:(?:" SingleChar R"(|\((\?:)?)" SingleChar R"(\)(?:\{1\}|\{1,1\})?)))"
464​SuggestPattern suggestPatterns[]{
465​ {"DetectChar", QStringLiteral("^" CharReg "(?:{1(?:,1)?\\})?$"), SuggestPattern::HasInsensitiveAttribute},
466​ {"Detect2Chars", QStringLiteral("^" CharReg CharReg "(?:\\{1(?:,1)?)?\\}$|^" CharReg "(?:{2(?:,2)?\\})$"), SuggestPattern::NoFlag},
467​ {"AnyChar", QStringLiteral(R"(^\^?\[(?:\\\]|[^\]^])(?:\\\]|[^\]])*\](?:\|\[(?:\\\]|[^\]^])(?:\\\]|[^\]])*\])*\$?$)"), SuggestPattern::HasInsensitiveAttribute},
468​ // (`C` | `[aA]`)* for "[sS][tT][rR]"
469​ {"StringDetect", QStringLiteral("^(?:" CharReg "|\\[[a-zA-Z]{2}\\])*$"), SuggestPattern::HasInsensitiveAttribute | SuggestPattern::HasDynamicAttribute},
470​ {"WordDetect", QStringLiteral(R"(^\\b)" CharReg R"(*\\b$)"), SuggestPattern::HasInsensitiveAttribute},
471​ {"WordDetect", QStringLiteral("^" CharReg R"(*\\b$)"), SuggestPattern::HasInsensitiveAttribute | SuggestPattern::OnlyIfColumn0},
472​ {"DetectIdentifier", QStringLiteral(R"(^\[(?:_|a-z){2}\]\[(?:(?:0-9|a-z|_){3}|\\w)\]\*$)"), SuggestPattern::NoFlag},
473​ {"DetectIdentifier", QStringLiteral(R"(^\[(?:_|a-z|A-Z){3}\]\[(?:(?:0-9|a-z|A-Z|_){4}|\\w)\]\*$)"), SuggestPattern::HasInsensitiveAttribute},
474​ {"DetectSpace", QStringLiteral(R"(^\\s\*?\$?$)"), SuggestPattern::HasInsensitiveAttribute},
475​ {"LineContinue", QStringLiteral("^" CharReg "\\$$"), SuggestPattern::NoFlag},
476​ {"HlCOct", QStringLiteral(R"(^0(?:\[0-7\]|\[01234567\])[+*]$)"), SuggestPattern::HasInsensitiveAttribute},
477​ {"HlCHex", QStringLiteral(R"(^0x\[(?:0-9|a-f|A-F){3}\][*+]$)"), SuggestPattern::NoFlag},
478​ {"HlCHex", QStringLiteral(R"(^0x\[(?:0-9|a-f|A-F){2,3}\][*+]$)"), SuggestPattern::HasInsensitiveAttribute},
479​ {"HlCStringChar", QStringLiteral(R"(^\\\[[abefnrtv"'?\\]{11,}\]$)"), SuggestPattern::NoFlag},
480​ {"HlCChar", QStringLiteral(R"(^'?(?:\((?:\?:)?)?(?:(?:)"
481​ R"('?\[\^\\\\\]'?|)"
482​ R"('?\\\\\[[abefnrtv"?\\']{12,}\]'?|)"
483​ R"('?\\\\x\[(?:0-9|a-f|(?:A-F)?)+\](?:\{2(?:,2)?\})?'?|)"
484​ R"('?\\\\0(?:\[0-7\]|\[01234567\])+(?:\{2(?:,2)?\})?'?)"
485​ R"()\|?){3,}\)?'?$)"), SuggestPattern::NoFlag},
486​ {"Int", QStringLiteral(R"(^(?:\[0-9\]|\\d)(?:\+|(?:(?:\[0-9\]|\\d)\*))$)"), SuggestPattern::NoFlag},
487​ {"RangeDetect", QStringLiteral("^(" CharReg R"()\[\^\1\]\*\1$)"), SuggestPattern::NoFlag},
488​};
489​#undef CharReg
490​#undef SingleChar
491​#undef EscapedChar
492​
493​//! Proposes to replace RegExpr with another element (HlCHex, HlCStringChar, RangeDetect, etc)
494​bool suggestForRegExp(const QString &hlFilename, QXmlStreamReader &xml)
495​{
496​ if (xml.name() == QLatin1String("RegExpr")) {
497​ auto string = xml.attributes().value(QLatin1String("String")).toString();
498​ if (xml.attributes().hasAttribute(QLatin1String("lookAhead"))) {
499​ const auto lookAhead = xml.attributes().value(QLatin1String("lookAhead"));
500​ if (string == QLatin1String("^$") || (Xml::attrToBool(lookAhead) && string == QLatin1String("."))) {
501​ const auto context = xml.attributes().value(QLatin1String("context"));
502​ qWarning() << hlFilename << "line" << xml.lineNumber()
503​ << QStringLiteral("RegExpr candidate for fallthroughContext=\"") + context + QStringLiteral("\" fallthrough=\"true\"");
504​ return false;
505​ }
506​ }
507​
508​ const QString originalString = string;
509​ const bool isInsensitive = Xml::attrToBool(xml.attributes().value(QLatin1String("insensitive")));
510​ const bool isDynamic = Xml::attrToBool(xml.attributes().value(QLatin1String("dynamic")));
511​ const bool asFirstNonSpace = string.startsWith(QLatin1String("^\\s*"));
512​ if (asFirstNonSpace) {
513​ string.remove(0, 4);
514​ }
515​ const bool asColumn0 = string.startsWith(QLatin1String("^"));
516​ if (asColumn0) {
517​ string.remove(0, 1);
518​ }
519​ const auto extraMessage = (asFirstNonSpace ? "with firstNonSpace=\"true\":" : (asColumn0 ? "with column=\"0\":" : ":"));
520​ for (const auto &suggestPattern : suggestPatterns) {
521​ if (isInsensitive && !bool(suggestPattern.flags & SuggestPattern::HasInsensitiveAttribute)) {
522​ continue;
523​ }
524​ if (bool(suggestPattern.flags & SuggestPattern::OnlyIfColumn0) && xml.attributes().value(QLatin1String("column")) != QLatin1String("0")) {
525​ continue;
526​ }
527​ if (!bool(suggestPattern.flags & SuggestPattern::HasDynamicAttribute) && isDynamic) {
528​ if (string.contains(QLatin1String("%1")) || string.contains(QLatin1String("%2"))) {
529​ continue;
530​ }
531​ }
532​ if (suggestPattern.regex.match(string).hasMatch()) {
533​ qWarning() << hlFilename << "line" << xml.lineNumber() << "RegExpr candidate for" << suggestPattern.suggest << extraMessage << originalString;
534​ return false;
535​ }
536​ }
537​ }
538​
539​ return true;
540​}
541​
542​/**
543​ * Helper class that suggests rule mergers
544​ */
545​class RuleMergeSuggest
546​{
547​public:
548​ bool suggest(const QString &hlFilename, QXmlStreamReader &xml)
549​ {
550​ if (xml.attributes().hasAttribute(QLatin1String("attribute")) && xml.name() != QLatin1String("context")) {
551​ const std::size_t specificAttributeLength = 3;
552​ const QStringRef values[] {
553​ xml.attributes().value(QLatin1String("String")),
554​ xml.attributes().value(QLatin1String("char")),
555​ xml.attributes().value(QLatin1String("char1")),
556​ xml.name(),
557​ xml.attributes().value(QLatin1String("attribute")),
558​ xml.attributes().value(QLatin1String("context")),
559​ xml.attributes().value(QLatin1String("beginRegion")),
560​ xml.attributes().value(QLatin1String("endRegion")),
561​ Xml::attrToBool(xml.attributes().value(QLatin1String("lookAhead"))) ? QStringRef(&m_true) : QStringRef(&m_false),
562​ Xml::attrToBool(xml.attributes().value(QLatin1String("firstNonSpace"))) ? QStringRef(&m_true) : QStringRef(&m_false),
563​ xml.attributes().value(QLatin1String("column")),
564​ };
565​ // check the size of the tables and display the number of elements in a compilation error in case of mismatch
566​ std::extent<decltype(m_previousRule)>::type{} = std::extent<decltype(values)>::type{};
567​
568​ if (std::equal(std::begin(values) + specificAttributeLength, std::end(values), std::begin(m_previousRule) + specificAttributeLength)
569​ && (canBeMerged(m_previousRule, values)
570​ || canBeMerged(values, m_previousRule)
571​ || std::equal(std::begin(values), std::end(values) - specificAttributeLength, std::begin(m_previousRule))
572​ )) {
573​ qWarning() << hlFilename << "line" << xml.lineNumber() << xml.name() << "can be merged with the previous rule";
574​ return false;
575​ }
576​
577​ std::size_t i = 0;
578​ for (auto& stringRef : values) {
579​ m_previousRule[i] = stringRef.toString();
580​ ++i;
581​ }
582​ }
583​ else {
584​ m_previousRule[3].clear();
585​ }
586​
587​ return true;
588​ }
589​
590​private:
591​ template<class Array1, class Array2>
592​ bool canBeMerged(const Array1 &array1, const Array2 &array2) {
593​ if (array1[3] == QLatin1String("RegExpr")) {
594​ return true;
595​ }
596​ if (array1[3] == QLatin1String("DetectChar")) {
597​ if (array2[3] == QLatin1String("AnyChar")) {
598​ return true;
599​ }
600​ if (array2[3] == QLatin1String("RangeDetect")) {
601​ const auto cleft = array2[1][0].digitValue();
602​ const auto cright = array2[2][0].digitValue();
603​ const auto c = array1[1][0].digitValue();
604​ return ((cleft <= c && c <= cright) || (c + 1 == cleft) || (cright + 1 == c));
605​ }
606​ }
607​ return false;
608​ }
609​
610​ QString m_previousRule[11];
611​ QString m_true = QStringLiteral("1");
612​ QString m_false = QStringLiteral("0");
613​};
614​
378} 615​}
379 616​
380int main(int argc, char *argv[]) 617​int main(int argc, char *argv[])
381{ 618​{
382 // get app instance 619​ // get app instance
383 QCoreApplication app(argc, argv); 620​ QCoreApplication app(argc, argv);
384 621​
385 // ensure enough arguments are passed 622​ // ensure enough arguments are passed
▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Line(s)687 Q_FOREACH (const QString &attribute, textAttributes) {
451 hl[attribute] = xml.attributes().value(attribute).toString(); 688​ hl[attribute] = xml.attributes().value(attribute).toString();
452 } 689​ }
453 690​
454 // numerical attributes 691​ // numerical attributes
455 hl[QStringLiteral("version")] = xml.attributes().value(QLatin1String("version")).toInt(); 692​ hl[QStringLiteral("version")] = xml.attributes().value(QLatin1String("version")).toInt();
456 hl[QStringLiteral("priority")] = xml.attributes().value(QLatin1String("priority")).toInt(); 693​ hl[QStringLiteral("priority")] = xml.attributes().value(QLatin1String("priority")).toInt();
457 694​
458 // add boolean one 695​ // add boolean one
459 const QString hidden = xml.attributes().value(QLatin1String("hidden")).toString(); 696​ hl[QStringLiteral("hidden")] = Xml::attrToBool( xml.attributes().value(QLatin1String("hidden")));
460 hl[QStringLiteral("hidden")] = (hidden == QLatin1String("true") || hidden == QLatin1String("1")); ​
461 697​
462 // remember hl 698​ // remember hl
463 hls[QFileInfo(hlFile).fileName()] = hl; 699​ hls[QFileInfo(hlFile).fileName()] = hl;
464 700​
465 AttributeChecker attributeChecker(hlFilename); 701​ AttributeChecker attributeChecker(hlFilename);
466 KeywordChecker keywordChecker(hlFilename); 702​ KeywordChecker keywordChecker(hlFilename);
703​ RuleMergeSuggest ruleMergeSuggest;
467 const QString hlName = hl[QStringLiteral("name")].toString(); 704​ const QString hlName = hl[QStringLiteral("name")].toString();
468 705​
469 // scan for broken regex or keywords with spaces 706​ // scan for broken regex or keywords with spaces
470 while (!xml.atEnd()) { 707​ while (!xml.atEnd()) {
471 xml.readNext(); 708​ xml.readNext();
472 if (!xml.isStartElement()) { 709​ if (!xml.isStartElement()) {
473 continue; 710​ continue;
474 } 711​ }
Show All 25 Lines735 if (!checkSingleChars(hlFilename, xml)) {
500 continue; 737​ continue;
501 } 738​ }
502 739​
503 // scan for lookAhead="true" with context="#stay" 740​ // scan for lookAhead="true" with context="#stay"
504 if (!checkLookAhead(hlFilename, xml)) { 741​ if (!checkLookAhead(hlFilename, xml)) {
505 anyError = 7; 742​ anyError = 7;
506 continue; 743​ continue;
507 } 744​ }
745​
746​ // suggestions for StringDetect
747​ if (!suggestForStringDetect(hlFilename, xml)) {
748​ //anyError = 12;
749​ continue;
750​ }
751​
752​ // suggestions for AnyChar
753​ if (!suggestForAnyChar(hlFilename, xml)) {
754​ //anyError = 12;
755​ continue;
756​ }
757​
758​ // suggestions for RegExp
759​ if (!suggestForRegExp(hlFilename, xml)) {
760​ //anyError = 12;
761​ continue;
762​ }
763​
764​ // suggestions for a rule merge
765​ if (ruleMergeSuggest.suggest(hlFilename, xml)) {
766​ //anyError = 12;
767​ continue;
768​ }
508 } 769​ }
509 770​
510 if (!attributeChecker.check()) { 771​ if (!attributeChecker.check()) {
511 anyError = 7; 772​ anyError = 7;
512 } 773​ }
513 774​
514 if (!keywordChecker.check()) { 775​ if (!keywordChecker.check()) {
515 anyError = 7; 776​ anyError = 7;
Show All 22 Lines