diff --git a/bin/EmergeConfig.py b/bin/EmergeConfig.py --- a/bin/EmergeConfig.py +++ b/bin/EmergeConfig.py @@ -23,7 +23,6 @@ emergeSettings = None - class EmergeStandardDirs( object ): __pathCache = dict( ) __noShortPathCache = dict( ) @@ -62,7 +61,9 @@ @staticmethod def allowShortpaths( allowd ): + old = EmergeStandardDirs._allowShortpaths EmergeStandardDirs._allowShortpaths = allowd + return old @staticmethod def downloadDir( ): @@ -149,9 +150,8 @@ def __init__( self ): self._config = None - EmergeStandardDirs.allowShortpaths( False ) - self.iniPath = os.path.join( EmergeStandardDirs.etcDir( ), "kdesettings.ini" ) - EmergeStandardDirs.allowShortpaths( True ) + with TemporaryUseShortpath(False): + self.iniPath = os.path.join( EmergeStandardDirs.etcDir( ), "kdesettings.ini" ) self._alias = dict( ) self._readSettings( ) @@ -248,6 +248,27 @@ self._config.write( configfile ) +class TemporaryUseShortpath(object): + """Context handler for temporarily different shortpath setting""" + def __init__(self, enabled): + self.prev = EmergeStandardDirs.allowShortpaths(enabled) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, trback): + EmergeStandardDirs.allowShortpaths(self.prev) + + + emergeSettings = EmergeConfig( ) + + + + + + + + diff --git a/bin/EmergeDebug.py b/bin/EmergeDebug.py --- a/bin/EmergeDebug.py +++ b/bin/EmergeDebug.py @@ -37,6 +37,19 @@ return Verbose.__level +class TemporaryVerbosity(object): + """Context handler for temporarily different verbosity""" + def __init__(self, tempLevel): + self.prevLevel = verbose() + setVerbose(tempLevel) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, trback): + setVerbose(self.prevLevel) + + def verbose(): """return the value of the verbose level""" return Verbose.level() diff --git a/bin/EmergeHash.py b/bin/EmergeHash.py new file mode 100644 --- /dev/null +++ b/bin/EmergeHash.py @@ -0,0 +1,101 @@ +from enum import Enum +import os +import hashlib +import re + +import EmergeDebug + + +class HashAlgorithm(Enum): + SHA1 = 1 + SHA224 = 2 + SHA256 = 3 + SHA512 = 4 + MD5 = 5 # deprecated + + def stringPrefix(self): + return "[%s]" % self.name + + def fileEnding(self): + return "." + self.name.lower() + + @classmethod + def fileEndings(cls): + out = dict() + for _, val in cls.__members__.items(): + out[val] = val.fileEnding() + return out + + @classmethod + def getAlgorithmFromFile(cls, file): + _, ext = os.path.splitext() + return cls.__getattr__(ext[1:].upper()) + + @classmethod + def getAlgorithmFromPrefix(cls, hash): + for alg in re.findall("\[.*\]", hash): + return cls.__getattr__(alg[1:-1]) + return None + + +def digestFile(filepath, algorithm=HashAlgorithm.SHA256): + """ digests a file """ + blockSize = 65536 + with open(filepath, "rb") as hashFile: + hash = getattr(hashlib, algorithm.name.lower())() + buffer = hashFile.read(blockSize) + while len(buffer) > 0: + hash.update(buffer) + buffer = hashFile.read(blockSize) + return hash.hexdigest() + + +def checkFilesDigests(downloaddir, filenames, digests=None, digestAlgorithm=HashAlgorithm.SHA1): + """check digest of (multiple) files specified by 'filenames' from 'downloaddir'""" + if type(digests) == list: + digestList = digests + else: + digestList = [digests] + + for digests in digestList: + for filename in filenames: + EmergeDebug.debug("checking digest of: %s" % filename, 1) + pathName = os.path.join(downloaddir, filename) + if digests == None: + for digestAlgorithm, digestFileEnding in HashAlgorithm.fileEndings().items(): + digestFileName = pathName + digestFileEnding + if not os.path.exists(digestFileName): + digestFileName, _ = os.path.splitext(pathName) + digestFileName += digestFileEnding + if not os.path.exists(digestFileName): + continue + currentHash = digestFile(pathName, digestAlgorithm) + with open(digestFileName, "rt+") as f: + data = f.read() + if not re.findall(currentHash, data): + EmergeDebug.error("%s hash for file %s (%s) does not match (%s)" % ( + digestAlgorithm.name, pathName, currentHash, data)) + return False + # digest provided in digests parameter + else: + currentHash = digestFile(pathName, digestAlgorithm) + if len(digests) != len(currentHash) or digests.find(currentHash) == -1: + EmergeDebug.error("%s hash for file %s (%s) does not match (%s)" % ( + digestAlgorithm.name, pathName, currentHash, digests)) + return False + return True + + +def createDigestFile(path, algorithm=HashAlgorithm.SHA1): + """creates a sha1 diget file""" + digets = digestFile(path, algorithm) + with open(path + algorithm.fileEnding(), "wt+") as f: + f.write("%s\n" % digets) + + +def printFilesDigests(downloaddir, filenames, buildTarget, algorithm=HashAlgorithm.SHA256): + out = "self.targetDigests['%s'] = ([" % buildTarget + for filename in filenames: + out += "'%s'," % digestFile(os.path.join(downloaddir, filename), algorithm) + print(out[:-1] + "], EmergeHash.%s)" % (algorithm)) + diff --git a/bin/EmergeSetupHelper.py b/bin/EmergeSetupHelper.py --- a/bin/EmergeSetupHelper.py +++ b/bin/EmergeSetupHelper.py @@ -49,12 +49,11 @@ subprocess.getoutput( command ) if emergeSettings.getboolean( "ShortPath", "EMERGE_USE_SHORT_PATH", False ): - EmergeStandardDirs.allowShortpaths( False ) - _subst( EmergeStandardDirs.emergeRoot( ), "EMERGE_ROOT_DRIVE" ) - _subst( EmergeStandardDirs.downloadDir( ), "EMERGE_DOWNLOAD_DRIVE" ) - _subst( EmergeStandardDirs.svnDir( ), "EMERGE_SVN_DRIVE" ) - _subst( EmergeStandardDirs.gitDir( ), "EMERGE_GIT_DRIVE" ) - EmergeStandardDirs.allowShortpaths( True ) + with TemporaryUseShortpath( False): + _subst( EmergeStandardDirs.emergeRoot( ), "EMERGE_ROOT_DRIVE" ) + _subst( EmergeStandardDirs.downloadDir( ), "EMERGE_DOWNLOAD_DRIVE" ) + _subst( EmergeStandardDirs.svnDir( ), "EMERGE_SVN_DRIVE" ) + _subst( EmergeStandardDirs.gitDir( ), "EMERGE_GIT_DRIVE" ) def printBanner( self ): print( "KDEROOT : %s" % EmergeStandardDirs.emergeRoot( ), file = sys.stderr ) diff --git a/bin/InstallDB.py b/bin/InstallDB.py --- a/bin/InstallDB.py +++ b/bin/InstallDB.py @@ -117,9 +117,8 @@ def __init__( self, filename = None ): if filename == None: - EmergeStandardDirs.allowShortpaths(False) - filename = os.path.join( EmergeStandardDirs.etcPortageDir(), 'install.db' ) - EmergeStandardDirs.allowShortpaths(True) + with TemporaryUseShortpath(False): + filename = os.path.join( EmergeStandardDirs.etcPortageDir(), 'install.db' ) self.dbfilename = filename self._prepareDatabase() @@ -252,28 +251,27 @@ def _prepareDatabase( self ): """ prepare a new database and add the required table layout """ - if not os.path.exists( self.dbfilename ): - EmergeStandardDirs.allowShortpaths(False) - if not os.path.exists( EmergeStandardDirs.etcPortageDir( ) ): - os.makedirs( EmergeStandardDirs.etcPortageDir( ) ) - print( "database does not exist yet: creating database", file = sys.stderr ) - self.connection = sqlite3.connect( self.dbfilename ) - cursor = self.connection.cursor() - - # first, create the required tables - cursor.execute( '''CREATE TABLE packageList (packageId INTEGER PRIMARY KEY AUTOINCREMENT, - prefix TEXT, category TEXT, packageName TEXT, version TEXT, revision TEXT)''' ) - cursor.execute( '''CREATE TABLE fileList (fileId INTEGER PRIMARY KEY AUTOINCREMENT, - packageId INTEGER, filename TEXT, fileHash TEXT)''' ) - self.connection.commit() - else: - self.connection = sqlite3.connect( self.dbfilename ) - cursor = self.connection.cursor() - cursor.execute( '''PRAGMA table_info('packageList')''') - if not len(cursor.fetchall()) == 6: - cursor.execute('''ALTER TABLE packageList ADD COLUMN revision TEXT''') - self.connection.commit() - EmergeStandardDirs.allowShortpaths(True) + with TemporaryUseShortpath(False): + if not os.path.exists( self.dbfilename ): + if not os.path.exists( EmergeStandardDirs.etcPortageDir( ) ): + os.makedirs( EmergeStandardDirs.etcPortageDir( ) ) + print( "database does not exist yet: creating database", file = sys.stderr ) + self.connection = sqlite3.connect( self.dbfilename ) + cursor = self.connection.cursor() + + # first, create the required tables + cursor.execute( '''CREATE TABLE packageList (packageId INTEGER PRIMARY KEY AUTOINCREMENT, + prefix TEXT, category TEXT, packageName TEXT, version TEXT, revision TEXT)''' ) + cursor.execute( '''CREATE TABLE fileList (fileId INTEGER PRIMARY KEY AUTOINCREMENT, + packageId INTEGER, filename TEXT, fileHash TEXT)''' ) + self.connection.commit() + else: + self.connection = sqlite3.connect( self.dbfilename ) + cursor = self.connection.cursor() + cursor.execute( '''PRAGMA table_info('packageList')''') + if not len(cursor.fetchall()) == 6: + cursor.execute('''ALTER TABLE packageList ADD COLUMN revision TEXT''') + self.connection.commit() # get a global object diff --git a/bin/Packager/NullsoftInstallerPackager.py b/bin/Packager/NullsoftInstallerPackager.py --- a/bin/Packager/NullsoftInstallerPackager.py +++ b/bin/Packager/NullsoftInstallerPackager.py @@ -5,6 +5,7 @@ from winreg import * # pylint: disable=F0401 import EmergeDebug +import EmergeHash import utils from Packager.CollectionPackagerBase import * @@ -141,5 +142,5 @@ self.internalCreatePackage() self.preArchive() self.generateNSISInstaller() - utils.createDigestFile( self.defines[ "setupname" ]) + EmergeHash.createDigestFile(self.defines["setupname"]) return True diff --git a/bin/Packager/PortablePackager.py b/bin/Packager/PortablePackager.py --- a/bin/Packager/PortablePackager.py +++ b/bin/Packager/PortablePackager.py @@ -2,6 +2,7 @@ # copyright (c) 2011 Hannah von Reth # import EmergeDebug +import EmergeHash import utils from .CollectionPackagerBase import * from .SevenZipPackager import * @@ -40,5 +41,5 @@ self.internalCreatePackage() self.createPortablePackage() - utils.createDigestFile( os.path.join(self.packageDestinationDir(), self.defines[ "setupname" ])) + EmergeHash.createDigestFile(os.path.join(self.packageDestinationDir(), self.defines["setupname"])) return True diff --git a/bin/Source/ArchiveSource.py b/bin/Source/ArchiveSource.py --- a/bin/Source/ArchiveSource.py +++ b/bin/Source/ArchiveSource.py @@ -6,6 +6,7 @@ import EmergeDebug from Source.SourceBase import * +import EmergeHash class ArchiveSource(SourceBase): @@ -46,18 +47,20 @@ def __checkFilesPresent(self, filenames): """check if all files for the current target are available""" - available = True for filename in filenames: path = os.path.join(EmergeStandardDirs.downloadDir(), filename) if self.subinfo.hasTargetDigests(): if not os.path.exists(path): - available = False + return False elif self.subinfo.hasTargetDigestUrls(): - if not os.path.exists("%s.sha1" % path): - available = False + algorithm = EmergeHash.HashAlgorithm.SHA1 + if type(self.subinfo.targetDigestUrl()) == tuple: + _, algorithm = self.subinfo.targetDigestUrl() + if not os.path.exists(path + algorithm.fileEnding()): + return False elif not os.path.exists(path): - available = False - return available + return False + return True def fetch( self, dummyRepopath = None ): """fetch normal tarballs""" @@ -80,8 +83,11 @@ EmergeDebug.debug("failed to download files", 1) return False if result and self.subinfo.hasTargetDigestUrls(): - if self.subinfo.targetDigestUrl() == "auto": - return utils.getFiles( self.subinfo.target(), EmergeStandardDirs.downloadDir(), ".sha1", self.subinfo.archiveName() ) + if type(self.subinfo.targetDigestUrl()) == tuple: + url, alg = self.subinfo.targetDigestUrl() + return utils.getFiles(url, EmergeStandardDirs.downloadDir(), + filenames = self.subinfo.archiveName() + + EmergeHash.HashAlgorithm.fileEndings().get(alg)) else: return utils.getFiles( self.subinfo.targetDigestUrl(), EmergeStandardDirs.downloadDir(), filenames = '' ) else: @@ -96,18 +102,20 @@ if self.subinfo.hasTargetDigestUrls(): EmergeDebug.debug("check digests urls", 1) - if not utils.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames): + if not EmergeHash.checkFilesDigests(EmergeStandardDirs.downloadDir(), filenames): EmergeDebug.error("invalid digest file") return False elif self.subinfo.hasTargetDigests(): EmergeDebug.debug("check digests", 1) - if not utils.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames, self.subinfo.targetDigest()): + digests = self.subinfo.targetDigest() + if type(digests) == tuple: + digests, algorithm = digests + if not EmergeHash.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames, digests, algorithm): EmergeDebug.error("invalid digest file") return False else: EmergeDebug.debug("print source file digests", 1) - digests = utils.createFilesDigests( EmergeStandardDirs.downloadDir(), filenames ) - utils.printFilesDigests( digests, self.subinfo.buildTarget) + EmergeHash.printFilesDigests(EmergeStandardDirs.downloadDir(), filenames, self.subinfo.buildTarget, algorithm = EmergeHash.HashAlgorithm.SHA256) return True def unpack(self): @@ -134,20 +142,7 @@ if hasattr(self.subinfo.options.unpack, 'unpackDir'): destdir = os.path.join(destdir, self.subinfo.options.unpack.unpackDir) - if self.subinfo.hasTargetDigestUrls(): - EmergeDebug.debug("check digests urls", 1) - if not utils.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames): - EmergeDebug.error("invalid digest file") - return False - elif self.subinfo.hasTargetDigests(): - EmergeDebug.debug("check digests", 1) - if not utils.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames, self.subinfo.targetDigest()): - EmergeDebug.error("invalid digest file") - return False - else: - EmergeDebug.debug("print source file digests", 1) - digests = utils.createFilesDigests( EmergeStandardDirs.downloadDir(), filenames ) - utils.printFilesDigests( digests, self.subinfo.buildTarget) + self.checkDigest() binEndings = (".exe", ".bat", ".msi") if (self.subinfo.archiveName() == "" and self.url.endswith(binEndings)) or self.subinfo.archiveName().endswith(binEndings): diff --git a/bin/dependencies.py b/bin/dependencies.py --- a/bin/dependencies.py +++ b/bin/dependencies.py @@ -630,7 +630,7 @@ def dumpDependencies(category, output_type=OUTPUT_DOT, dep_type="both"): """without displaying debuginfo in generated output""" - with utils.TemporaryVerbosity(0): + with EmergeDebug.TemporaryVerbosity(0): packageList, categoryList = portage.getPackagesCategories(category) dep_tree = DependenciesTree() for _category, _package in zip(categoryList, packageList): @@ -640,7 +640,7 @@ def dumpDependenciesForPackageList(packageList, output_type=OUTPUT_DOT, dep_type="both"): """without displaying debuginfo in generated output""" - with utils.TemporaryVerbosity(0): + with EmergeDebug.TemporaryVerbosity(0): dep_tree = DependenciesTree() for category, package, dummyTarget, dummyPatchlevel in packageList: dep_tree.addDependencies(category, package, dep_type=dep_type) diff --git a/bin/emerge.py b/bin/emerge.py --- a/bin/emerge.py +++ b/bin/emerge.py @@ -178,7 +178,6 @@ item.target = args.target EmergeDebug.debug("dependency: %s" % item, 1) - if not deplist: EmergeDebug.debug("", 1) @@ -453,7 +452,9 @@ except KeyboardInterrupt: pass except portage.PortageException as e: - EmergeDebug.debug(e.exception, 1) + if e.exception: + EmergeDebug.debug(e.exception, 0) + traceback.print_tb( e.exception.__traceback__) EmergeDebug.error(e) except Exception as e: print( e ) diff --git a/bin/info.py b/bin/info.py --- a/bin/info.py +++ b/bin/info.py @@ -10,6 +10,7 @@ import os from collections import OrderedDict +import EmergeDebug import utils import compiler from options import * @@ -110,7 +111,7 @@ if not buildTarget == None: self.buildTarget = buildTarget if not self.buildTarget in list(self.targets.keys()) and not self.buildTarget in list(self.svnTargets.keys()) : - utils.debug("build target %s not defined in available targets %s %s setting default target to %s" % (self.buildTarget, list(self.targets.keys()), list(self.svnTargets.keys()), self.defaultTarget), 1) + EmergeDebug.debug("build target %s not defined in available targets %s %s setting default target to %s" % (self.buildTarget, list(self.targets.keys()), list(self.svnTargets.keys()), self.defaultTarget), 1) self.buildTarget = self.defaultTarget def setBuildOptions( self ): @@ -214,7 +215,7 @@ """returns the archive file name""" if self.buildTarget in list(self.archiveNames.keys()): return self.archiveNames[self.buildTarget] - return "" + return os.path.split(self.targets[self.buildTarget])[-1] def hasMultipleTargets( self ): """return whether we used a list of targets""" @@ -285,7 +286,7 @@ if (self.buildTarget in list(self.targets.keys()) or self.buildTarget in list(self.svnTargets.keys())) \ and self.buildTarget in list(self.targetInstallPath.keys()): return self.targetInstallPath[ self.buildTarget ] - utils.die("no install path for this build target defined") + EmergeDebug.die("no install path for this build target defined") def hasMergePath(self): """return true if relative path appendable to local merge path is given for the recent target""" diff --git a/bin/test/EmergeTestBase.py b/bin/test/EmergeTestBase.py new file mode 100644 --- /dev/null +++ b/bin/test/EmergeTestBase.py @@ -0,0 +1,18 @@ +import unittest +import tempfile +import os + +import EmergeConfig +import EmergeDebug + +class EmergeTestBase(unittest.TestCase): + + def setUp(self): + EmergeDebug.setVerbose(int(os.environ["EMERGE_TEST_VERBOSITY"])) + self.kdeRoot = tempfile.TemporaryDirectory() + EmergeConfig.EmergeStandardDirs.allowShortpaths(False) + EmergeConfig.EmergeStandardDirs._pathCache()["EMERGEROOT"] = self.kdeRoot.name + os.environ["KDEROOT"] = self.kdeRoot.name + + def tearDown(self): + del self.kdeRoot \ No newline at end of file diff --git a/bin/test/runtests.py b/bin/test/runtests.py --- a/bin/test/runtests.py +++ b/bin/test/runtests.py @@ -39,6 +39,13 @@ import sys import unittest import optparse +import tempfile + + +sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir)) + +import EmergeDebug +import EmergeConfig def main(): @@ -50,9 +57,11 @@ dest="verbosity") opts, rest = parser.parse_args() - os.environ["EMERGE_VERBOSE"] = str(opts.verbosity) - files = os.listdir(os.curdir) + EmergeDebug.setVerbose(opts.verbosity) + os.environ["EMERGE_TEST_VERBOSITY"] = str(opts.verbosity) + + files = os.listdir(os.path.dirname(__file__)) names = [] for file in files: if file[:4] == "test" and file[-3:] == ".py": diff --git a/bin/test/test_EmergeHash.py b/bin/test/test_EmergeHash.py new file mode 100644 --- /dev/null +++ b/bin/test/test_EmergeHash.py @@ -0,0 +1,61 @@ +import os +import sys +import unittest +import random +import tempfile +import contextlib +import io + +import EmergeHash +import EmergeDebug +import EmergeTestBase + +class EmergeHashTest(EmergeTestBase.EmergeTestBase): + + def setUp(self): + super().setUp() + random.seed(42) + data = "" + for i in range(0, 1000): + data += str(random.random()) + self.tmpFile = tempfile.NamedTemporaryFile("wt+", delete=False) + self.tmpFile.write(data) + self.tmpFile.close() + + + def tearDown(self): + del self.tmpFile + super().tearDown() + + + def hashTest(self, hash, algorithm): + path, name = os.path.split(self.tmpFile.name) + self.assertEquals(EmergeHash.checkFilesDigests(path, [name], hash, algorithm), True) + + +class TestAPI(EmergeHashTest): + + def test_MD5(self): + self.hashTest("953700da7dfea74714b08f8a7cf69151", EmergeHash.HashAlgorithm.MD5) + + def test_SHA1(self): + self.hashTest("25f0187fc5e189518dc489bcc97daa93973d7d1e", EmergeHash.HashAlgorithm.SHA1) + + def test_SHA224(self): + self.hashTest("deef4b9f1cf9e7bdfa34ba05521740a7fdf3e1570b704a3d4088cec2", EmergeHash.HashAlgorithm.SHA224) + + def test_SHA256(self): + self.hashTest("4fc1e96dc5ecf625efe228fce1b0964b6302cfa4d4fb2bb8d16c665d23f6ff30", EmergeHash.HashAlgorithm.SHA256) + + def test_SHA512(self): + self.hashTest( + "70f8f3087b51217d16e860915a06a5208eb51fb2264c10815d395feb834f63cc28fb9abed6c681b7475fbfb3dcd1afc713b16789ea951d27ab34e8d637cc27f4", + EmergeHash.HashAlgorithm.SHA512) + + def test_printFilesDigests(self): + path, name = os.path.split(self.tmpFile.name) + log = io.StringIO() + with contextlib.redirect_stdout(log): + EmergeHash.printFilesDigests(path, [name], "test", EmergeHash.HashAlgorithm.SHA256) + self.assertEquals("self.targetDigests['test'] = (['4fc1e96dc5ecf625efe228fce1b0964b6302cfa4d4fb2bb8d16c665d23f6ff30'], EmergeHash.HashAlgorithm.SHA256)\n", + log.getvalue()) \ No newline at end of file diff --git a/bin/test/test_InstallDB.py b/bin/test/test_InstallDB.py --- a/bin/test/test_InstallDB.py +++ b/bin/test/test_InstallDB.py @@ -37,24 +37,20 @@ import sys import unittest -test_dir = os.path.dirname(__file__) -sys.path.append(os.path.join(test_dir, os.pardir)) -os.environ["KDEROOT"] = test_dir - import InstallDB +import EmergeConfig +import EmergeTestBase -tmp_dir = os.path.join(test_dir, "tmp") -class DatabaseTest(unittest.TestCase): +class DatabaseTest(EmergeTestBase.EmergeTestBase): def setUp(self): - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) - self.db = InstallDB.InstallDB(os.path.join(tmp_dir, "test.db")) + super().setUp() + self.db = InstallDB.InstallDB(os.path.join(EmergeConfig.EmergeStandardDirs.etcDir(), "test.db")) def tearDown(self): - os.remove(os.path.join(tmp_dir, "test.db")) - os.rmdir(tmp_dir) + del self.db + super().tearDown() class TestAPI(DatabaseTest): diff --git a/bin/utils.py b/bin/utils.py --- a/bin/utils.py +++ b/bin/utils.py @@ -11,7 +11,6 @@ import ctypes import datetime import ftplib -import hashlib import http.client import inspect import shutil @@ -23,6 +22,7 @@ import zipfile from operator import itemgetter +import EmergeHash import Notifier.NotificationLoader from EmergeConfig import * from EmergeDebug import verbose, setVerbose, info, debug, warning, error, die @@ -73,20 +73,6 @@ debug("could not find application %s" % appname, 1) return False - -class TemporaryVerbosity(object): - """Context handler for temporarily different verbosity""" - def __init__(self, tempLevel): - self.prevLevel = verbose() - setVerbose(tempLevel) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, trback): - setVerbose(self.prevLevel) - - def getFiles( urls, destdir, suffix='' , filenames = ''): """download files from 'url' into 'destdir'""" debug("getfiles called. urls: %s, filenames: %s, suffix: %s" % (urls, filenames, suffix), 1) @@ -203,95 +189,6 @@ with open(filename, "rb") as f: return str(f.readline(),'UTF-8').endswith("\r\n") -def checkFilesDigests( downloaddir, filenames, digests=None ): - """check digest of (multiple) files specified by 'filenames' from 'downloaddir'""" - if digests != None: - if type(digests) == list: - digestList = digests - elif digests.find("\n") != -1: - digestList = digests.splitLines() - else: - digestList = [digests] - - i = 0 - for filename in filenames: - debug("checking digest of: %s" % filename, 1) - pathName = os.path.join( downloaddir, filename ) - if digests == None: - digestFileName = pathName + '.sha1' - if not os.path.exists( digestFileName ): - digestFileName, _ = os.path.splitext( pathName ) - digestFileName += '.sha1' - if not os.path.exists( digestFileName ): - error("digest validation request for file %s, but no digest file present" % - pathName) - return False - currentHash = digestFileSha1( pathName ) - with open( digestFileName, "r" ) as f: - line = f.readline() - digest = re.search('\\b[0-9a-fA-F]{40}\\b', line) - if not digest: - error(" digestFile %s for file %s does not contain a valid SHA1 hash" % (digestFileName, - pathName,)) - return False - digest = digest.group(0) - if len(digest) != len(currentHash) or digest.find(currentHash) == -1: - error("SHA1 hash for file %s (%s) does not match (%s)" % (pathName, currentHash, digest)) - return False - # digest provided in digests parameter - else: - currentHash = digestFileSha1( pathName ) - digest = digestList[i].strip() - if len(digest) != len(currentHash) or digest.find(currentHash) == -1: - error("SHA1 hash for file %s (%s) does not match (%s)" % (pathName, currentHash, digest)) - return False - i = i + 1 - return True - - -def createFilesDigests( downloaddir, filenames ): - """create digests of (multiple) files specified by 'filenames' from 'downloaddir'""" - digestList = list() - for filename in filenames: - pathName = os.path.join( downloaddir, filename ) - digest = digestFileSha1( pathName ) - entry = filename, digest - digestList.append(entry) - return digestList - -def createDigestFile(path): - """creates a sha1 diget file""" - digets = digestFileSha1(path) - with open(path + ".sha1","wt+") as f: - f.write("%s\n" % digets) - -def printFilesDigests( digestFiles, buildTarget=None): - size = len( digestFiles ) - i = 0 - for (fileName, digest) in digestFiles: - print("%40s %s" % ( fileName, digest ), end=' ') - if size == 1: - if buildTarget == None: - print(" '%s'" % ( digest )) - else: - print("self.targetDigests['%s'] = '%s'" % ( buildTarget, digest )) - else: - if buildTarget == None: - if i == 0: - print(" ['%s'," % ( digest )) - elif i == size-1: - print(" '%s']" % ( digest )) - else: - print(" '%s'," % ( digest )) - i = i + 1 - else: - if i == 0: - print("self.targetDigests['%s'] = ['%s'," % ( buildTarget, digest )) - elif i == size-1: - print(" '%s']" % ( digest )) - else: - print(" '%s'," % ( digest )) - i = i + 1 ### unpack functions @@ -468,18 +365,23 @@ if ( not imagedir.endswith( "\\" ) ): myimagedir = myimagedir + "\\" + algorithm = EmergeHash.HashAlgorithm.SHA256 for root, _, files in os.walk( imagedir ): for fileName in files: - ret.append( ( os.path.join( root, fileName ).replace( myimagedir, "" ), digestFile( os.path.join( root, fileName ) ) ) ) + ret.append( ( os.path.join( root, fileName ).replace( myimagedir, "" ), algorithm.stringPrefix() + EmergeHash.digestFile( os.path.join( root, fileName), algorithm) ) ) return ret def unmergeFileList(rootdir, fileList, forced=False): """ delete files in the fileList if has matches or forced is True """ for filename, filehash in fileList: fullPath = os.path.join(rootdir, os.path.normcase( filename)) if os.path.isfile(fullPath): - currentHash = digestFile(fullPath) + algorithm = EmergeHash.HashAlgorithm.getAlgorithmFromPrefix(filehash) + if not algorithm: + currentHash = EmergeHash.digestFile(fullPath, EmergeHash.HashAlgorithm.MD5) + else: + currentHash = algorithm.stringPrefix() + EmergeHash.digestFile(fullPath, algorithm) if currentHash == filehash or filehash == "": debug("deleting file %s" % fullPath, 2) try: @@ -604,26 +506,6 @@ finally: os.chdir( olddir ) -def digestFile( filepath ): - """ md5-digests a file """ - fileHash = hashlib.md5() - try: - with open( filepath, "rb" ) as digFile: - for line in digFile: - fileHash.update( line ) - return fileHash.hexdigest() - except IOError: - return "" - - -def digestFileSha1( filepath ): - """ sha1-digests a file """ - fileHash = hashlib.sha1() - with open( filepath, "rb" ) as hashFile: - for line in hashFile: - fileHash.update( line ) - return fileHash.hexdigest() - def getVCSType( url ): """ return the type of the vcs url """ if not url: diff --git a/portage/data/hicolor-icon-theme/hicolor-icon-theme.py b/portage/data/hicolor-icon-theme/hicolor-icon-theme.py --- a/portage/data/hicolor-icon-theme/hicolor-icon-theme.py +++ b/portage/data/hicolor-icon-theme/hicolor-icon-theme.py @@ -3,9 +3,8 @@ class subinfo(info.infoclass): def setTargets( self ): for v in [ '0.15' ]: - self.targets[v] = 'http://icon-theme.freedesktop.org/releases/hicolor-icon-theme-' + v + '.tar.xz' - self.targetDigests['0.15'] = '368ecc47521e1a2f72cef0171bee02aae876ff53' - + self.targets[v] = 'http://icon-theme.freedesktop.org/releases/hicolor-icon-theme-' + v + '.tar.xz' + self.targetDigests['0.15'] = (['9cc45ac3318c31212ea2d8cb99e64020732393ee7630fa6c1810af5f987033cc'], EmergeHash.HashAlgorithm.SHA256) self.shortDescription = "High-color icon theme shell from the FreeDesktop project" self.defaultTarget = '0.15' diff --git a/portage/dev-util/cmake/cmake.py b/portage/dev-util/cmake/cmake.py --- a/portage/dev-util/cmake/cmake.py +++ b/portage/dev-util/cmake/cmake.py @@ -1,15 +1,12 @@ import info - +import EmergeHash class subinfo( info.infoclass ): def setTargets( self ): for ver in ['3.2.1', '3.2.2', '3.2.3', '3.3.1', '3.4.1']: self.targets[ver] = 'http://www.cmake.org/files/v%s/cmake-%s-win32-x86.zip' % (ver[:3], ver) self.targetMergeSourcePath[ver] = 'cmake-%s-win32-x86' % ver - self.targetDigests['3.2.1'] = '4011f4f18c002a9ff97c76ea1d397eca9b675f98' - self.targetDigests['3.2.3'] = 'de3acd4c99057584bb2d149a982eca47caad8e22' - self.targetDigests['3.3.1'] = 'cbe93de9e5861c8b0b441d5c40fb68b9b27ac7af' - self.targetDigests['3.4.1'] = '4894baeafc0368d6530bf2c6bfe4fc94056bd04a' + self.targetDigestUrls[ver] = ("https://cmake.org/files/v%s/cmake-%s-SHA-256.txt"% (ver[:3], ver), EmergeHash.HashAlgorithm.SHA256) nightlyUrl = "https://cmake.org/files/dev/" nightlyVer = utils.getNightlyVersionsFromUrl(nightlyUrl + "?C=M;O=D;F=0", "\d.\d.\d\d\d\d\d\d\d\d-[0-9A-Za-z]{5,8}" + re.escape("-win32-x86"))[0]