Index: bin/EmergeDebug.py =================================================================== --- bin/EmergeDebug.py +++ bin/EmergeDebug.py @@ -37,6 +37,19 @@ return Verbose.__level +class TemporaryVerbosity(object): + """Context handler for temporarily different verbosity""" + def __init__(self, tempLevel): + self.prevLevel = verbose() + setVerbose(tempLevel) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, trback): + setVerbose(self.prevLevel) + + def verbose(): """return the value of the verbose level""" return Verbose.level() Index: bin/EmergeHash.py =================================================================== --- /dev/null +++ bin/EmergeHash.py @@ -0,0 +1,100 @@ +from enum import Enum +import os +import hashlib +import re + +import EmergeDebug + +class HashAlgorithm(Enum): + SHA1 = 1 + SHA224 = 2 + SHA256 = 3 + SHA512 = 4 + MD5 = 5# deprecated + + def stringPrefix(self): + return "[%s]" % self.name + + def fileEnding(self): + return "." + self.name.lower() + + @classmethod + def fileEndings(cls): + out = dict() + for _,val in cls.__members__.items(): + out[val] = val.fileEnding() + return out + + @classmethod + def getAlgorithmFromFile(cls, file): + _,ext = os.path.splitext() + return cls.__getattr__(ext[1:].upper()) + + @classmethod + def getAlgorithmFromPrefix(cls,hash): + for alg in re.findall("\[.*\]", hash): + return cls.__getattr__(alg[1:-1]) + return None + +def digestFile(filepath, algorithm = HashAlgorithm.SHA256): + """ digests a file """ + blockSize = 1024 + with open( filepath, "rb" ) as hashFile: + hash = getattr(hashlib, algorithm.name.lower())() + buffer = hashFile.read(blockSize) + while len(buffer)>0: + hash.update(buffer) + buffer = hashFile.read(blockSize) + return hash.hexdigest() + +def checkFilesDigests(downloaddir, filenames, digests=None, digestAlgorithm = HashAlgorithm.SHA1): + """check digest of (multiple) files specified by 'filenames' from 'downloaddir'""" + if type(digests) == list: + digestList = digests + else: + digestList = [digests] + + for digests in digestList: + for filename in filenames: + EmergeDebug.debug( "checking digest of: %s" % filename, 1 ) + pathName = os.path.join( downloaddir, filename ) + if digests == None: + for digestAlgorithm, digestFileEnding in HashAlgorithm.fileEndings().items(): + digestFileName = pathName + digestFileEnding + if not os.path.exists( digestFileName ): + digestFileName, _ = os.path.splitext( pathName ) + digestFileName += digestFileEnding + if not os.path.exists( digestFileName ): + continue + currentHash = digestFile( pathName, digestAlgorithm) + with open( digestFileName, "rt+" ) as f: + data = f.read() + if not re.findall(currentHash, data): + EmergeDebug.error( "%s hash for file %s (%s) does not match (%s)" % (digestAlgorithm.name, pathName, currentHash, data) ) + return False + # digest provided in digests parameter + else: + currentHash = digestFile( pathName, digestAlgorithm) + if len(digests) != len(currentHash) or digests.find(currentHash) == -1: + EmergeDebug.error( "%s hash for file %s (%s) does not match (%s)" % (digestAlgorithm.name, pathName, currentHash, digests) ) + return False + return True + +def createDigestFile(path, algorithm = HashAlgorithm.SHA1): + """creates a sha1 diget file""" + digets = digestFile(path, algorithm) + with open(path + algorithm.fileEnding(),"wt+") as f: + f.write("%s\n" % digets) + + +def printFilesDigests(downloaddir, filenames, buildTarget, algorithm = HashAlgorithm.SHA256): + digestList = list() + for filename in filenames: + pathName = os.path.join( downloaddir, filename ) + digest = digestFile( pathName, algorithm) + digestList.append(digest) + out = "self.targetDigests['%s'] = ([" % buildTarget + for digest in digestList: + out += "'%s'," % ( digest ) + print(out[:-1] + "], EmergeHash.%s)" % ( algorithm )) + Index: bin/Packager/NullsoftInstallerPackager.py =================================================================== --- bin/Packager/NullsoftInstallerPackager.py +++ bin/Packager/NullsoftInstallerPackager.py @@ -5,6 +5,7 @@ from winreg import * # pylint: disable=F0401 import EmergeDebug +import EmergeHash import utils from Packager.CollectionPackagerBase import * @@ -141,5 +142,5 @@ self.internalCreatePackage() self.preArchive() self.generateNSISInstaller() - utils.createDigestFile( self.defines[ "setupname" ]) + EmergeHash.createDigestFile(self.defines["setupname"]) return True Index: bin/Packager/PortablePackager.py =================================================================== --- bin/Packager/PortablePackager.py +++ bin/Packager/PortablePackager.py @@ -2,6 +2,7 @@ # copyright (c) 2011 Hannah von Reth # import EmergeDebug +import EmergeHash import utils from .CollectionPackagerBase import * from .SevenZipPackager import * @@ -40,5 +41,5 @@ self.internalCreatePackage() self.createPortablePackage() - utils.createDigestFile( os.path.join(self.packageDestinationDir(), self.defines[ "setupname" ])) + EmergeHash.createDigestFile(os.path.join(self.packageDestinationDir(), self.defines["setupname"])) return True Index: bin/Source/ArchiveSource.py =================================================================== --- bin/Source/ArchiveSource.py +++ bin/Source/ArchiveSource.py @@ -6,6 +6,7 @@ import EmergeDebug from Source.SourceBase import * +import EmergeHash class ArchiveSource(SourceBase): @@ -46,18 +47,20 @@ def __checkFilesPresent(self, filenames): """check if all files for the current target are available""" - available = True for filename in filenames: path = os.path.join(EmergeStandardDirs.downloadDir(), filename) if self.subinfo.hasTargetDigests(): if not os.path.exists(path): - available = False + return False elif self.subinfo.hasTargetDigestUrls(): - if not os.path.exists("%s.sha1" % path): - available = False + algorithm = EmergeHash.HashAlgorithm.SHA1 + if type(self.subinfo.targetDigestUrl()) == tuple: + _, algorithm = self.subinfo.targetDigestUrl() + if not os.path.exists(path + algorithm.fileEnding()): + return False elif not os.path.exists(path): - available = False - return available + return False + return True def fetch( self, dummyRepopath = None ): """fetch normal tarballs""" @@ -80,8 +83,11 @@ EmergeDebug.debug("failed to download files", 1) return False if result and self.subinfo.hasTargetDigestUrls(): - if self.subinfo.targetDigestUrl() == "auto": - return utils.getFiles( self.subinfo.target(), EmergeStandardDirs.downloadDir(), ".sha1", self.subinfo.archiveName() ) + if type(self.subinfo.targetDigestUrl()) == tuple: + url, alg = self.subinfo.targetDigestUrl() + return utils.getFiles(url, EmergeStandardDirs.downloadDir(), + filenames = self.subinfo.archiveName() + + EmergeHash.HashAlgorithm.fileEndings().get(alg)) else: return utils.getFiles( self.subinfo.targetDigestUrl(), EmergeStandardDirs.downloadDir(), filenames = '' ) else: @@ -96,18 +102,20 @@ if self.subinfo.hasTargetDigestUrls(): EmergeDebug.debug("check digests urls", 1) - if not utils.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames): + if not EmergeHash.checkFilesDigests(EmergeStandardDirs.downloadDir(), filenames): EmergeDebug.error("invalid digest file") return False elif self.subinfo.hasTargetDigests(): EmergeDebug.debug("check digests", 1) - if not utils.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames, self.subinfo.targetDigest()): + digests = self.subinfo.targetDigest() + if type(digests) == tuple: + digests, algorithm = digests + if not EmergeHash.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames, digests, algorithm): EmergeDebug.error("invalid digest file") return False else: EmergeDebug.debug("print source file digests", 1) - digests = utils.createFilesDigests( EmergeStandardDirs.downloadDir(), filenames ) - utils.printFilesDigests( digests, self.subinfo.buildTarget) + EmergeHash.printFilesDigests(EmergeStandardDirs.downloadDir(), filenames, self.subinfo.buildTarget, algorithm = EmergeHash.HashAlgorithm.SHA256) return True def unpack(self): @@ -134,20 +142,7 @@ if hasattr(self.subinfo.options.unpack, 'unpackDir'): destdir = os.path.join(destdir, self.subinfo.options.unpack.unpackDir) - if self.subinfo.hasTargetDigestUrls(): - EmergeDebug.debug("check digests urls", 1) - if not utils.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames): - EmergeDebug.error("invalid digest file") - return False - elif self.subinfo.hasTargetDigests(): - EmergeDebug.debug("check digests", 1) - if not utils.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames, self.subinfo.targetDigest()): - EmergeDebug.error("invalid digest file") - return False - else: - EmergeDebug.debug("print source file digests", 1) - digests = utils.createFilesDigests( EmergeStandardDirs.downloadDir(), filenames ) - utils.printFilesDigests( digests, self.subinfo.buildTarget) + self.checkDigest() binEndings = (".exe", ".bat", ".msi") if (self.subinfo.archiveName() == "" and self.url.endswith(binEndings)) or self.subinfo.archiveName().endswith(binEndings): Index: bin/dependencies.py =================================================================== --- bin/dependencies.py +++ bin/dependencies.py @@ -630,7 +630,7 @@ def dumpDependencies(category, output_type=OUTPUT_DOT, dep_type="both"): """without displaying debuginfo in generated output""" - with utils.TemporaryVerbosity(0): + with EmergeDebug.TemporaryVerbosity(0): packageList, categoryList = portage.getPackagesCategories(category) dep_tree = DependenciesTree() for _category, _package in zip(categoryList, packageList): @@ -640,7 +640,7 @@ def dumpDependenciesForPackageList(packageList, output_type=OUTPUT_DOT, dep_type="both"): """without displaying debuginfo in generated output""" - with utils.TemporaryVerbosity(0): + with EmergeDebug.TemporaryVerbosity(0): dep_tree = DependenciesTree() for category, package, dummyTarget, dummyPatchlevel in packageList: dep_tree.addDependencies(category, package, dep_type=dep_type) Index: bin/emerge.py =================================================================== --- bin/emerge.py +++ bin/emerge.py @@ -176,7 +176,6 @@ item.target = args.target EmergeDebug.debug("dependency: %s" % item, 1) - if not deplist: EmergeDebug.debug("", 1) @@ -450,7 +449,9 @@ except KeyboardInterrupt: pass except portage.PortageException as e: - EmergeDebug.debug(e.exception, 1) + if e.exception: + EmergeDebug.debug(e.exception, 0) + traceback.print_tb( e.exception.__traceback__) EmergeDebug.error(e) except Exception as e: print( e ) Index: bin/info.py =================================================================== --- bin/info.py +++ bin/info.py @@ -10,6 +10,7 @@ import os from collections import OrderedDict +import EmergeDebug import utils import compiler from options import * @@ -110,7 +111,7 @@ if not buildTarget == None: self.buildTarget = buildTarget if not self.buildTarget in list(self.targets.keys()) and not self.buildTarget in list(self.svnTargets.keys()) : - utils.debug("build target %s not defined in available targets %s %s setting default target to %s" % (self.buildTarget, list(self.targets.keys()), list(self.svnTargets.keys()), self.defaultTarget), 1) + EmergeDebug.debug("build target %s not defined in available targets %s %s setting default target to %s" % (self.buildTarget, list(self.targets.keys()), list(self.svnTargets.keys()), self.defaultTarget), 1) self.buildTarget = self.defaultTarget def setBuildOptions( self ): @@ -214,7 +215,7 @@ """returns the archive file name""" if self.buildTarget in list(self.archiveNames.keys()): return self.archiveNames[self.buildTarget] - return "" + return os.path.split(self.targets[self.buildTarget])[-1] def hasMultipleTargets( self ): """return whether we used a list of targets""" @@ -285,7 +286,7 @@ if (self.buildTarget in list(self.targets.keys()) or self.buildTarget in list(self.svnTargets.keys())) \ and self.buildTarget in list(self.targetInstallPath.keys()): return self.targetInstallPath[ self.buildTarget ] - utils.die("no install path for this build target defined") + EmergeDebug.die("no install path for this build target defined") def hasMergePath(self): """return true if relative path appendable to local merge path is given for the recent target""" Index: bin/utils.py =================================================================== --- bin/utils.py +++ bin/utils.py @@ -11,7 +11,6 @@ import ctypes import datetime import ftplib -import hashlib import http.client import inspect import shutil @@ -23,6 +22,7 @@ import zipfile from operator import itemgetter +import EmergeHash import Notifier.NotificationLoader from EmergeConfig import * from EmergeDebug import verbose, setVerbose, info, debug, warning, error, die @@ -73,20 +73,6 @@ debug("could not find application %s" % appname, 1) return False - -class TemporaryVerbosity(object): - """Context handler for temporarily different verbosity""" - def __init__(self, tempLevel): - self.prevLevel = verbose() - setVerbose(tempLevel) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, trback): - setVerbose(self.prevLevel) - - def getFiles( urls, destdir, suffix='' , filenames = ''): """download files from 'url' into 'destdir'""" debug("getfiles called. urls: %s, filenames: %s, suffix: %s" % (urls, filenames, suffix), 1) @@ -203,95 +189,6 @@ with open(filename, "rb") as f: return str(f.readline(),'UTF-8').endswith("\r\n") -def checkFilesDigests( downloaddir, filenames, digests=None ): - """check digest of (multiple) files specified by 'filenames' from 'downloaddir'""" - if digests != None: - if type(digests) == list: - digestList = digests - elif digests.find("\n") != -1: - digestList = digests.splitLines() - else: - digestList = [digests] - - i = 0 - for filename in filenames: - debug("checking digest of: %s" % filename, 1) - pathName = os.path.join( downloaddir, filename ) - if digests == None: - digestFileName = pathName + '.sha1' - if not os.path.exists( digestFileName ): - digestFileName, _ = os.path.splitext( pathName ) - digestFileName += '.sha1' - if not os.path.exists( digestFileName ): - error("digest validation request for file %s, but no digest file present" % - pathName) - return False - currentHash = digestFileSha1( pathName ) - with open( digestFileName, "r" ) as f: - line = f.readline() - digest = re.search('\\b[0-9a-fA-F]{40}\\b', line) - if not digest: - error(" digestFile %s for file %s does not contain a valid SHA1 hash" % (digestFileName, - pathName,)) - return False - digest = digest.group(0) - if len(digest) != len(currentHash) or digest.find(currentHash) == -1: - error("SHA1 hash for file %s (%s) does not match (%s)" % (pathName, currentHash, digest)) - return False - # digest provided in digests parameter - else: - currentHash = digestFileSha1( pathName ) - digest = digestList[i].strip() - if len(digest) != len(currentHash) or digest.find(currentHash) == -1: - error("SHA1 hash for file %s (%s) does not match (%s)" % (pathName, currentHash, digest)) - return False - i = i + 1 - return True - - -def createFilesDigests( downloaddir, filenames ): - """create digests of (multiple) files specified by 'filenames' from 'downloaddir'""" - digestList = list() - for filename in filenames: - pathName = os.path.join( downloaddir, filename ) - digest = digestFileSha1( pathName ) - entry = filename, digest - digestList.append(entry) - return digestList - -def createDigestFile(path): - """creates a sha1 diget file""" - digets = digestFileSha1(path) - with open(path + ".sha1","wt+") as f: - f.write("%s\n" % digets) - -def printFilesDigests( digestFiles, buildTarget=None): - size = len( digestFiles ) - i = 0 - for (fileName, digest) in digestFiles: - print("%40s %s" % ( fileName, digest ), end=' ') - if size == 1: - if buildTarget == None: - print(" '%s'" % ( digest )) - else: - print("self.targetDigests['%s'] = '%s'" % ( buildTarget, digest )) - else: - if buildTarget == None: - if i == 0: - print(" ['%s'," % ( digest )) - elif i == size-1: - print(" '%s']" % ( digest )) - else: - print(" '%s'," % ( digest )) - i = i + 1 - else: - if i == 0: - print("self.targetDigests['%s'] = ['%s'," % ( buildTarget, digest )) - elif i == size-1: - print(" '%s']" % ( digest )) - else: - print(" '%s'," % ( digest )) - i = i + 1 ### unpack functions @@ -468,18 +365,23 @@ if ( not imagedir.endswith( "\\" ) ): myimagedir = myimagedir + "\\" + algorithm = EmergeHash.HashAlgorithm.SHA256 for root, _, files in os.walk( imagedir ): for fileName in files: - ret.append( ( os.path.join( root, fileName ).replace( myimagedir, "" ), digestFile( os.path.join( root, fileName ) ) ) ) + ret.append( ( os.path.join( root, fileName ).replace( myimagedir, "" ), algorithm.stringPrefix() + EmergeHash.digestFile( os.path.join( root, fileName), algorithm) ) ) return ret def unmergeFileList(rootdir, fileList, forced=False): """ delete files in the fileList if has matches or forced is True """ for filename, filehash in fileList: fullPath = os.path.join(rootdir, os.path.normcase( filename)) if os.path.isfile(fullPath): - currentHash = digestFile(fullPath) + algorithm = EmergeHash.HashAlgorithm.getAlgorithmFromPrefix(filehash) + if not algorithm: + currentHash = EmergeHash.digestFile(fullPath, EmergeHash.HashAlgorithm.MD5) + else: + currentHash = algorithm.stringPrefix() + EmergeHash.digestFile(fullPath, algorithm) if currentHash == filehash or filehash == "": debug("deleting file %s" % fullPath, 2) try: @@ -604,26 +506,6 @@ finally: os.chdir( olddir ) -def digestFile( filepath ): - """ md5-digests a file """ - fileHash = hashlib.md5() - try: - with open( filepath, "rb" ) as digFile: - for line in digFile: - fileHash.update( line ) - return fileHash.hexdigest() - except IOError: - return "" - - -def digestFileSha1( filepath ): - """ sha1-digests a file """ - fileHash = hashlib.sha1() - with open( filepath, "rb" ) as hashFile: - for line in hashFile: - fileHash.update( line ) - return fileHash.hexdigest() - def getVCSType( url ): """ return the type of the vcs url """ if not url: Index: portage/data/hicolor-icon-theme/hicolor-icon-theme.py =================================================================== --- portage/data/hicolor-icon-theme/hicolor-icon-theme.py +++ portage/data/hicolor-icon-theme/hicolor-icon-theme.py @@ -3,9 +3,8 @@ class subinfo(info.infoclass): def setTargets( self ): for v in [ '0.15' ]: - self.targets[v] = 'http://icon-theme.freedesktop.org/releases/hicolor-icon-theme-' + v + '.tar.xz' - self.targetDigests['0.15'] = '368ecc47521e1a2f72cef0171bee02aae876ff53' - + self.targets[v] = 'http://icon-theme.freedesktop.org/releases/hicolor-icon-theme-' + v + '.tar.xz' + self.targetDigests['0.15'] = (['9cc45ac3318c31212ea2d8cb99e64020732393ee7630fa6c1810af5f987033cc'], EmergeHash.HashAlgorithm.SHA256) self.shortDescription = "High-color icon theme shell from the FreeDesktop project" self.defaultTarget = '0.15' Index: portage/dev-util/cmake/cmake.py =================================================================== --- portage/dev-util/cmake/cmake.py +++ portage/dev-util/cmake/cmake.py @@ -1,15 +1,12 @@ import info - +import EmergeHash class subinfo( info.infoclass ): def setTargets( self ): for ver in ['3.2.1', '3.2.2', '3.2.3', '3.3.1', '3.4.1']: self.targets[ver] = 'http://www.cmake.org/files/v%s/cmake-%s-win32-x86.zip' % (ver[:3], ver) self.targetMergeSourcePath[ver] = 'cmake-%s-win32-x86' % ver - self.targetDigests['3.2.1'] = '4011f4f18c002a9ff97c76ea1d397eca9b675f98' - self.targetDigests['3.2.3'] = 'de3acd4c99057584bb2d149a982eca47caad8e22' - self.targetDigests['3.3.1'] = 'cbe93de9e5861c8b0b441d5c40fb68b9b27ac7af' - self.targetDigests['3.4.1'] = '4894baeafc0368d6530bf2c6bfe4fc94056bd04a' + self.targetDigestUrls[ver] = ("https://cmake.org/files/v%s/cmake-%s-SHA-256.txt"% (ver[:3], ver), EmergeHash.HashAlgorithm.SHA256) nightlyUrl = "https://cmake.org/files/dev/" nightlyVer = utils.getNightlyVersionsFromUrl(nightlyUrl + "?C=M;O=D", "\d.\d.\d\d\d\d\d\d\d\d-[0-9A-Za-z]{5,8}")[0]