Changeset View
Standalone View
thumbnail/textcreator.cpp
| Show All 28 Lines | |||||
| 29 | #include <QTextCodec> | 29 | #include <QTextCodec> | ||
| 30 | #include <QTextDocument> | 30 | #include <QTextDocument> | ||
| 31 | 31 | | |||
| 32 | #include <KSyntaxHighlighting/SyntaxHighlighter> | 32 | #include <KSyntaxHighlighting/SyntaxHighlighter> | ||
| 33 | #include <KSyntaxHighlighting/Theme> | 33 | #include <KSyntaxHighlighting/Theme> | ||
| 34 | #include <KSyntaxHighlighting/Definition> | 34 | #include <KSyntaxHighlighting/Definition> | ||
| 35 | #include <KDesktopFile> | 35 | #include <KDesktopFile> | ||
| 36 | 36 | | |||
| 37 | #include <config-thumbnail.h> | ||||
| 38 | #if LIBMAGIC_FOUND | ||||
sitter: TBH, I would make libmagic required for building the thumbnail plugin. I can't see much of a… | |||||
Without libmagic, it is current state basically UTF-8 with bom detection otherwise local codec. I did not test exhaustive encodings so I wanted to let the door open for users to not rely on libmagic. And libmagic loads a 5M file storing its heuristics each time it loads ( /usr/share/misc/magic.mgc ). meven: Without libmagic, it is current state basically UTF-8 with bom detection otherwise local codec. | |||||
Perhaps it'd make sense to refactor this a bit and construct some test cases around encoding detection so we get a sense of reliablity? The way I am looking at this: either libmagic always does the best job at detecting encodings, at which point we'll want it as a required dep, or there's something better in which case we don't want libmagic at all and instead use the something better ;) In the end the user isn't necessarily in charge of what a random file will be encoded with, so I don't think there's a point in letting the user (or the distro) build an inferior product by accidentally not including libmagic. The truth is neither we nor the user can with any certainty say what encodings the thumbnailer will encounter. sitter: Perhaps it'd make sense to refactor this a bit and construct some test cases around encoding… | |||||
| 39 | #include "magic.h" | ||||
| 40 | #endif | ||||
| 41 | | ||||
| 37 | // TODO Fix or remove kencodingprober code | 42 | // TODO Fix or remove kencodingprober code | ||
| 38 | // #include <kencodingprober.h> | 43 | // #include <kencodingprober.h> | ||
| 39 | 44 | | |||
| 40 | extern "C" | 45 | extern "C" | ||
| 41 | { | 46 | { | ||
| 42 | Q_DECL_EXPORT ThumbCreator *new_creator() | 47 | Q_DECL_EXPORT ThumbCreator *new_creator() | ||
| 43 | { | 48 | { | ||
| 44 | return new TextCreator; | 49 | return new TextCreator; | ||
| 45 | } | 50 | } | ||
| 46 | } | 51 | } | ||
| 47 | 52 | | |||
| 48 | TextCreator::TextCreator() | 53 | TextCreator::TextCreator() | ||
| 49 | : m_data(nullptr), | 54 | : m_data(nullptr), | ||
| 50 | m_dataSize(0) | 55 | m_dataSize(0) | ||
| 51 | { | 56 | { | ||
| 52 | } | 57 | } | ||
| 53 | 58 | | |||
| 54 | TextCreator::~TextCreator() | 59 | TextCreator::~TextCreator() | ||
| 55 | { | 60 | { | ||
| 56 | delete [] m_data; | 61 | delete [] m_data; | ||
| 57 | } | 62 | } | ||
| 58 | 63 | | |||
| 59 | static QTextCodec *codecFromContent(const char *data, int dataSize) | 64 | #if LIBMAGIC_FOUND | ||
| 65 | static QTextCodec* codecFromFile(const QString &path) | ||||
sitter: `*` on wrong side of space | |||||
| 66 | { | ||||
| 67 | magic_t m = magic_open(MAGIC_MIME_ENCODING); | ||||
sitter: better name than m? (: | |||||
| 68 | magic_load(m, nullptr); | ||||
| 69 | const char *ret = magic_file(m, path.toLocal8Bit() ); | ||||
excess whitespace towards the end. I also wonder if qfile::encodename would be better sitter: excess whitespace towards the end. I also wonder if qfile::encodename would be better | |||||
| 70 | auto codecName = QString(ret).toUpper().toLocal8Bit(); | ||||
I guess you could just toUpper on a QBA instead of going through a temporary QString since ret is an encoding identifier ajnd would be always an ascii string. sitter: I guess you could just toUpper on a QBA instead of going through a temporary QString since ret… | |||||
| 71 | magic_close(m); | ||||
| 72 | | ||||
| 73 | if (QTextCodec::availableCodecs().contains(codecName)) { | ||||
| 74 | return QTextCodec::codecForName(codecName); | ||||
| 75 | } | ||||
| 76 | | ||||
| 77 | if (strcmp(ret, "unknown-8bit")) { | ||||
| 78 | // use latin for unkwnwn 8bit as it is quite versatile | ||||
sitter: unkwnwn typo | |||||
| 79 | return QTextCodec::codecForName("latin-1"); | ||||
| 80 | } | ||||
| 81 | | ||||
| 82 | return nullptr; | ||||
| 83 | } | ||||
| 84 | #endif | ||||
| 85 | | ||||
| 86 | static QTextCodec *codecFromContent(const char *data, int dataSize, const QString &path) | ||||
| 60 | { | 87 | { | ||
| 61 | #if 0 // ### Use this when KEncodingProber does not return junk encoding for UTF-8 data) | | |||
| 62 | KEncodingProber prober; | | |||
| 63 | prober.feed(data, dataSize); | | |||
| 64 | return QTextCodec::codecForName(prober.encoding()); | | |||
| 65 | #else | | |||
| 66 | QByteArray ba = QByteArray::fromRawData(data, dataSize); | 88 | QByteArray ba = QByteArray::fromRawData(data, dataSize); | ||
| 67 | // try to detect UTF text, fall back to locale default (which is usually UTF-8) | 89 | // try to detect UTF text, fall back to locale default (which is usually UTF-8) | ||
| 68 | return QTextCodec::codecForUtfText(ba, QTextCodec::codecForLocale()); | 90 | const auto codec = QTextCodec::codecForUtfText(ba, nullptr); | ||
| 91 | | ||||
| 92 | if (codec == nullptr) { | ||||
| 93 | // UTF-8 BOM detection failed | ||||
| 94 | auto localCodec = QTextCodec::codecForLocale(); | ||||
| 95 | #if LIBMAGIC_FOUND | ||||
| 96 | if (localCodec->name() == "UTF-8") { | ||||
| 97 | // UTF-8 was already tested in QTextCodec::codecForUtfText | ||||
| 98 | // but only using bom presence | ||||
| 99 | // The file could be utf-8 still or something entirely different | ||||
| 100 | // use libmagic heuristics | ||||
| 101 | auto codecMagic = codecFromFile(path); | ||||
| 102 | if (codecMagic != nullptr) { | ||||
| 103 | return codecMagic; | ||||
| 104 | } | ||||
| 105 | } | ||||
| 69 | #endif | 106 | #endif | ||
| 107 | return localCodec; | ||||
| 108 | } | ||||
| 109 | return codec; | ||||
| 70 | } | 110 | } | ||
| 71 | 111 | | |||
| 72 | bool TextCreator::create(const QString &path, int width, int height, QImage &img) | 112 | bool TextCreator::create(const QString &path, int width, int height, QImage &img) | ||
| 73 | { | 113 | { | ||
| 74 | // Desktop files, .directory files, and flatpakrefs aren't traditional | 114 | // Desktop files, .directory files, and flatpakrefs aren't traditional | ||
| 75 | // text files, so their icons should be shown instead | 115 | // text files, so their icons should be shown instead | ||
| 76 | if (KDesktopFile::isDesktopFile(path) | 116 | if (KDesktopFile::isDesktopFile(path) | ||
| 77 | || path.endsWith(QStringLiteral(".directory")) | 117 | || path.endsWith(QStringLiteral(".directory")) | ||
| ▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Line(s) | 158 | { | |||
| 121 | m_dataSize = bytesToRead + 1; | 161 | m_dataSize = bytesToRead + 1; | ||
| 122 | } | 162 | } | ||
| 123 | 163 | | |||
| 124 | int read = file.read( m_data, bytesToRead ); | 164 | int read = file.read( m_data, bytesToRead ); | ||
| 125 | if ( read > 0 ) | 165 | if ( read > 0 ) | ||
| 126 | { | 166 | { | ||
| 127 | ok = true; | 167 | ok = true; | ||
| 128 | m_data[read] = '\0'; | 168 | m_data[read] = '\0'; | ||
| 129 | QString text = codecFromContent( m_data, read )->toUnicode( m_data, read ).trimmed(); | 169 | QString text = codecFromContent(m_data, read, path)->toUnicode(m_data, read).trimmed(); | ||
| 130 | // FIXME: maybe strip whitespace and read more? | 170 | // FIXME: maybe strip whitespace and read more? | ||
| 131 | 171 | | |||
| 132 | // If the text contains tabs or consecutive spaces, it is probably | 172 | // If the text contains tabs or consecutive spaces, it is probably | ||
| 133 | // formatted using white space. Use a fixed pitch font in this case. | 173 | // formatted using white space. Use a fixed pitch font in this case. | ||
| 134 | const auto textLines = text.splitRef(QLatin1Char('\n')); | 174 | const auto textLines = text.splitRef(QLatin1Char('\n')); | ||
| 135 | for (const auto& line : textLines) { | 175 | for (const auto& line : textLines) { | ||
| 136 | const auto trimmedLine = line.trimmed(); | 176 | const auto trimmedLine = line.trimmed(); | ||
| 137 | if ( trimmedLine.contains( '\t' ) || trimmedLine.contains( " " ) ) { | 177 | if ( trimmedLine.contains( '\t' ) || trimmedLine.contains( " " ) ) { | ||
| ▲ Show 20 Lines • Show All 53 Lines • Show Last 20 Lines | |||||
TBH, I would make libmagic required for building the thumbnail plugin. I can't see much of a rationale for why we'd want to support "broken"/insufficient encoding detection when there's code that makes it better.