Differential D19996 Diff 54663 kde-modules/httpcheck.py

Changeset View

Standalone View

kde-modules/httpcheck.py

This file was added.

Property	Old Value	New Value
File Mode	null	100755

		1		#!/usr/bin/env python3
		2
		3		# Copyright 2019 Sandro Knauß <sknauss@kde.org>
		4		# Copyright 2019 Volker Krause <vkrause@kde.org>
		5		#
		6		# Redistribution and use in source and binary forms, with or without
		7		# modification, are permitted provided that the following conditions
		8		# are met:
		9		#
		10		# 1. Redistributions of source code must retain the copyright
		11		# notice, this list of conditions and the following disclaimer.
		12		# 2. Redistributions in binary form must reproduce the copyright
		13		# notice, this list of conditions and the following disclaimer in the
		14		# documentation and/or other materials provided with the distribution.
		15		# 3. The name of the author may not be used to endorse or promote products
		16		# derived from this software without specific prior written permission.
		17		#
		18		# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
		19		# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
		20		# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
		21		# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
		22		# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
		23		# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
		24		# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
		25		# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
		26		# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
		27		# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
		28
		29		import fnmatch
		30		import glob
		31		import os
		32		import re
		33		import sys
		34
		35		class HTTPChecker:
		36		def __init__(self, blacklist):
		37		self.blacklist = blacklist
		38		self.reBlacklist = list(map(lambda r:re.compile(r,re.I), blacklist))
		39		self.matcher = re.compile(r'http://\S', re.I)
		40
		41		def checkFile(self, fpath):
		42		findings = []
		43		with open(fpath, 'r') as f:
		44		if fpath.startswith('./'):
		45		fpath = fpath[2:]
		46		linenumber = 0
		47		for line in f:
		48		linenumber += 1
		49		m = self.matcher.search(line)
		50		if m:
		51		if m.group(0)[-1] == ".":
		52		continue
		53		if not any(map(lambda r: r.search(line), self.reBlacklist)):
		54		print("{}:{}:\t{}".format(fpath,linenumber,line.strip()))
		55		findings.append((linenumber, line))
		56		return findings
		57
		58		def loadBlacklist(path, ignoreAuto=False):
		59		blacklist = []
		60		try:
		61		with open(path) as f:
		62		comment = ""
		63		for line in f:
		64		m = re.match(r"^\s#\s(.*)", line)
		65		if m:
		66		comment = m.group(1).strip()
		67		continue
		68
		69		if ignoreAuto and comment.startswith("Auto:"):
		70		comment = ""
		71		continue
		72
		73		if line.strip():
		74		blacklist.append(line.strip())
		75		comment = ""
		76		except FileNotFoundError:
		77		pass
		78		return blacklist
		79
		80		def loadBlacklist(path, ignoreAuto=False):
		81		blacklist = []
		82		try:
		83		with open(path) as f:
		84		comment = ""
		85		for line in f:
		86		m = re.match(r"^\s#\s(.*)", line)
		87		if m:
		88		comment = m.group(1).strip()
		89		continue
		90
		91		if ignoreAuto and comment.startswith("Auto:"):
		92		comment = ""
		93		continue
		94
		95		if line.strip():
		96		blacklist.append(line.strip())
		97		comment = ""
		98		except FileNotFoundError:
		99		pass
		100		return blacklist
		101
		102		def getBlacklist(path, ignoreAuto=False):
		103		blacklist = []
		104		for fname in glob.glob(os.path.join(os.path.dirname(__file__), "*.htignore")):
		105		if ignoreAuto and fname.endswith('reduce-warning.htignore'):
		106		continue
		107		blacklist += loadBlacklist(fname, ignoreAuto)
		108		return blacklist + loadBlacklist(path, ignoreAuto)
		109
		110		class GitIgnore:
		111		def __init__(self, path):
		112		self.basepath = path
		113		self.patterns = []
		114		with open(os.path.join(path, '.gitignore')) as f:
		115		for line in f:
		116		self.patterns.append(line.strip())
		117
		118		def match(self, path):
		119		for pattern in self.patterns:
		120		if fnmatch.fnmatch(os.path.relpath(path, self.basepath), pattern):
		121		return True
		122		return False
		123
		124		def main(path):
		125		blacklist = getBlacklist(os.path.join(path, '.htignore'))
		126		checker = HTTPChecker(blacklist)
		127		gitIgnore = GitIgnore(path)
		128		gitIgnore.patterns.append('.htignore')
		129		gitIgnore.patterns.append('.gitignore')
		130		findings = {}
		131		for dirpath, dirnames, filenames in os.walk(path):
		132		parts = dirpath.split("/")
		133		if any(map(lambda p: p in parts, ['.git', 'tests', 'autotests', '3rdparty'])):
		134		continue
		135		for fname in filenames:
		136		fpath = os.path.join(dirpath, fname)
		137		if gitIgnore.match(fpath):
		138		continue
		139		try:
		140		issues = checker.checkFile(fpath)
		141		if issues:
		142		findings[fpath] = issues
		143		except UnicodeDecodeError:
		144		pass
		145		if findings:
		146		sys.exit(1)
		147		else:
		148		sys.exit(0)
		149
		150		if __name__ == "__main__":
		151		path = "."
		152		if len(sys.argv) > 1:
		153		path = sys.argv[1]
		154		urls = main(path)