Changeset View
Changeset View
Standalone View
Standalone View
pology/catalog.py
Show First 20 Lines • Show All 78 Lines • ▼ Show 20 Line(s) | 67 | if lcache: | |||
---|---|---|---|---|---|
79 | self._lines_msgstr = [] | 79 | self._lines_msgstr = [] | ||
80 | 80 | | |||
81 | 81 | | |||
82 | def _read_lines_and_encoding (file, filename): | 82 | def _read_lines_and_encoding (file, filename): | ||
83 | 83 | | |||
84 | fstr = file.read() | 84 | fstr = file.read() | ||
85 | # Determine line ending. | 85 | # Determine line ending. | ||
86 | maxlno = 0 | 86 | maxlno = 0 | ||
87 | for clend in ("\r\n", "\n", "\r"): # "\r\n" should be checked first | 87 | for clend in (b"\r\n", b"\n", b"\r"): # "\r\n" should be checked first | ||
88 | lno = len(fstr.split(clend)) | 88 | lno = len(fstr.split(clend)) | ||
89 | if maxlno < lno: | 89 | if maxlno < lno: | ||
90 | maxlno = lno | 90 | maxlno = lno | ||
91 | lend = clend | 91 | lend = clend | ||
92 | lines = [x + "\n" for x in fstr.split(lend)] | 92 | lines = [x + b"\n" for x in fstr.split(lend)] | ||
93 | if lines[-1] == "\n": | 93 | if lines[-1] == b"\n": | ||
94 | lines.pop() | 94 | lines.pop() | ||
95 | 95 | | |||
96 | enc = None | 96 | enc = None | ||
97 | enc_rx = re.compile(r"Content-Type:.*charset=(.+?)\\n", re.I) | 97 | enc_rx = re.compile(rb"Content-Type:.*charset=(.+?)\\n", re.I) | ||
98 | for line in lines: | 98 | for line in lines: | ||
99 | if line.strip().startswith("#:"): | 99 | if line.strip().startswith(b"#:"): | ||
100 | break | 100 | break | ||
101 | m = enc_rx.search(line) | 101 | m = enc_rx.search(line) | ||
102 | if m: | 102 | if m: | ||
103 | enc = m.group(1).strip() | 103 | enc = m.group(1).strip() | ||
104 | if not enc or enc == "CHARSET": # no encoding given | 104 | if not enc or enc == b"CHARSET": # no encoding given | ||
105 | enc = None | 105 | enc = None | ||
106 | break | 106 | break | ||
107 | if enc is None: | 107 | if enc is None: | ||
108 | enc = "UTF-8" # fall back to UTF-8 if encoding not found | 108 | enc = b"UTF-8" # fall back to UTF-8 if encoding not found | ||
109 | enc = enc.decode() | ||||
109 | 110 | | |||
110 | enclines = [] | 111 | enclines = [] | ||
111 | lno = 0 | 112 | lno = 0 | ||
112 | for line in lines: | 113 | for line in lines: | ||
113 | lno += 1 | 114 | lno += 1 | ||
114 | try: | 115 | try: | ||
115 | encline = line.decode(enc) | 116 | encline = line.decode(enc) | ||
116 | except UnicodeDecodeError as e: | 117 | except UnicodeDecodeError as e: | ||
▲ Show 20 Lines • Show All 2415 Lines • Show Last 20 Lines |