diff --git a/bin/Package/PackageBase.py b/bin/Package/PackageBase.py index 33247f4d8..607116b17 100644 --- a/bin/Package/PackageBase.py +++ b/bin/Package/PackageBase.py @@ -1,248 +1,274 @@ # # copyright (c) 2009 Ralf Habacker # import EmergeDebug from EmergeBase import * from InstallDB import * from compiler import * class PackageBase (EmergeBase): """ provides a generic interface for packages and implements the basic stuff for all packages """ # uses the following instance variables # todo: place in related ...Base #rootdir -> EmergeBase #package -> PackageBase #force -> PackageBase #category -> PackageBase #version -> PackageBase #packagedir -> PackageBase #imagedir -> PackageBase def __init__(self): EmergeDebug.debug("PackageBase.__init__ called", 2) EmergeBase.__init__(self) def _installedDBPrefix(self, buildType=None): postfix = '' if buildType == None: buildType = self.buildType() if self.useBuildTypeRelatedMergeRoot: if buildType == 'Debug': postfix = 'debug' elif buildType == 'Release': postfix = 'release' elif buildType == 'RelWithDebInfo': postfix = 'relwithdebinfo' return postfix def qmerge( self ): """mergeing the imagedirectory into the filesystem""" ## \todo is this the optimal place for creating the post install scripts ? ignoreInstalled = False prefixPath = self._installedDBPrefix( self.buildType() ) if installdb.isInstalled( category=None, package=self.package, prefix=prefixPath ): ignoreInstalled = True self.unmerge() - + EmergeDebug.debug("qmerge package to %s" % self.mergeDestinationDir(), 2) utils.mergeImageDirToRootDir( self.mergeSourceDir(), self.mergeDestinationDir() ) # run post-install scripts if not emergeSettings.getboolean("General","EMERGE_NO_POST_INSTALL", False ): for pkgtype in ['bin', 'lib', 'doc', 'src']: scriptName = "post-install-%s-%s.cmd" % ( self.package, pkgtype ) script = os.path.join( self.mergeDestinationDir(), "manifest", scriptName ) if os.path.exists( script ): EmergeDebug.debug("run post install script '%s'" % script, 2) cmd = "cd /D %s && %s" % ( self.mergeDestinationDir(), script ) if not utils.system(cmd): EmergeDebug.warning("%s failed!" % cmd) else: EmergeDebug.debug("post install script '%s' not found" % script, 2) else: EmergeDebug.debug("running of post install scripts disabled!") # add package to installed database -> is this not the task of the manifest files ? # only packages using a specific merge destination path are shared between build types revision = self.sourceRevision() if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \ and self.subinfo.options.merge.destinationPath != None: for prefix in [ "Release", "RelWithDebInfo", "Debug" ]: package = installdb.addInstalled( self.category, self.package, self.version, self._installedDBPrefix( prefix ), ignoreInstalled, revision = revision) package.addFiles( utils.getFileListFromDirectory( self._installedDBPrefix( prefix ) ) ) package.install() else: package = installdb.addInstalled( self.category, self.package, self.version, self._installedDBPrefix(), ignoreInstalled, revision = revision ) package.addFiles( utils.getFileListFromDirectory( self.mergeSourceDir() ) ) package.install() return True def unmerge( self ): """unmergeing the files from the filesystem""" EmergeDebug.debug("Packagebase unmerge called", 2) ## \todo mergeDestinationDir() reads the real used merge dir from the ## package definition, which fails if this is changed ## a better solution will be to save the merge sub dir into ## /etc/portage/installed and to read from it on unmerge EmergeDebug.debug("unmerge package from %s" % self.mergeDestinationDir(), 2) if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \ and self.subinfo.options.merge.destinationPath != None: for prefix in [ "Release", "RelWithDebInfo", "Debug" ]: packageList = installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( prefix ) ) for package in packageList: fileList = package.getFilesWithHashes() utils.unmergeFileList( self.mergeDestinationDir(), fileList, self.forced ) package.uninstall() else: packageList = installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( ) ) for package in packageList: fileList = package.getFilesWithHashes() utils.unmergeFileList( self.mergeDestinationDir(), fileList, self.forced ) package.uninstall() # only packages using a specific merge destination path are shared between build types if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \ and self.subinfo.options.merge.destinationPath != None: installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "Release" ) ) installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "RelWithDebInfo" ) ) installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "Debug" ) ) else: installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( ) ) # run post-uninstall scripts if not emergeSettings.getboolean("General","EMERGE_NO_POST_INSTALL", False ): for pkgtype in ['bin', 'lib', 'doc', 'src']: scriptName = "post-uninstall-%s-%s.cmd" % ( self.package, pkgtype ) script = os.path.join( self.mergeDestinationDir(), "manifest", scriptName ) if os.path.exists( script ): EmergeDebug.debug("run post uninstall script '%s'" % script, 2) cmd = "cd /D %s && %s" % ( self.mergeDestinationDir(), script ) if not utils.system(cmd): EmergeDebug.warning("%s failed!" % cmd) else: EmergeDebug.debug("post uninstall script '%s' not found" % script, 2) else: EmergeDebug.debug("running of post uninstall scripts disabled!") return True - def cleanImage( self ): + def cleanImage( self ) -> bool: """cleanup before install to imagedir""" if ( os.path.exists( self.imageDir() ) ): EmergeDebug.debug("cleaning image dir: %s" % self.imageDir(), 1) utils.cleanDirectory( self.imageDir() ) os.rmdir(self.imageDir()) return True - def cleanBuild( self ): + def cleanBuild( self ) -> bool: """cleanup currently used build dir""" if os.path.exists( self.buildDir() ): utils.cleanDirectory( self.buildDir() ) EmergeDebug.debug("cleaning build dir: %s" % self.buildDir(), 1) return True def stripLibs( self, pkgName ): """strip debugging informations from shared libraries - mingw only!!! """ - return self.strip(pkgName + ".dll" ) - + return self.strip(pkgName + ".dll" ) + def strip( self , fileName ): """strip debugging informations from shared libraries and executables - mingw only!!! """ - if self.subinfo.options.package.disableStriping or not isMinGW(): + if self.subinfo.options.package.disableStriping or not isMinGW(): EmergeDebug.debug("Skiping stipping of " + fileName, 2) return True basepath = os.path.join( self.installDir() ) filepath = os.path.join( basepath, "bin", fileName ) cmd = "strip -s " + filepath EmergeDebug.debug(cmd, 2) os.system( cmd ) return True def createImportLibs( self, pkgName ): """create the import libraries for the other compiler(if ANSI-C libs)""" basepath = os.path.join( self.installDir() ) utils.createImportLibs( pkgName, basepath ) def printFiles(self): packageList = installdb.getInstalledPackages(self.category, self.package, self._installedDBPrefix()) for package in packageList: fileList = package.getFiles() fileList.sort() for file in fileList: print(file[0]) return True def getAction(self, cmd = None ): if not cmd: command = sys.argv[ 1 ] options = None # print sys.argv if ( len( sys.argv ) > 2 ): options = sys.argv[ 2: ] else: command = cmd options = None # \todo options are not passed through by emerge.py fix it return [command, options] def execute( self, cmd=None ): """called to run the derived class this will be executed from the package if the package is started on its own it shouldn't be called if the package is imported as a python module""" EmergeDebug.debug("PackageBase.execute called. args: %s" % sys.argv, 2) command, _ = self.getAction(cmd) if self.subinfo.options.disableReleaseBuild and self.buildType() == "Release" \ or self.subinfo.options.disableDebugBuild and self.buildType() == "Debug": print("target ignored for this build type") return False return self.runAction(command) + + def binaryArchiveName(self, pkgSuffix = None): + if not pkgSuffix: + pkgSuffix = '' + if hasattr(self.subinfo.options.package, 'packageSuffix') and self.subinfo.options.package.packageSuffix: + pkgSuffix = self.subinfo.options.package.packageSuffix + + pkgVersion, _ = self.getPackageVersion() + if self.package.endswith( "-package" ): + shortPackage = self.package[ : -8 ] + else: + shortPackage = self.package + + return "%s-%s-%s%s%s.%s" % (shortPackage, compiler.architecture(), pkgVersion, compiler.getShortName(), pkgSuffix, emergeSettings.get("Packager", "7ZipArchiveType", "7z")) + + def fetchBinary(self) -> bool: + archiveName = self.binaryArchiveName() + downloadFolder = os.path.join(EmergeStandardDirs.downloadDir(), "binary") + if not os.path.exists(downloadFolder): + os.mkdir(downloadFolder) + return utils.getFile("%s/%s" % (emergeSettings.get("ContinuousIntegration", "BinaryUrl"), archiveName), downloadFolder )\ + and self.cleanImage()\ + and utils.unpackFile(downloadFolder, archiveName, self.imageDir())\ + and self.qmerge() + def runAction( self, command ): """ \todo TODO: rename the internal functions into the form cmdFetch, cmdCheckDigest etc then we get by without this dict: ok = getattr(self, 'cmd' + command.capitalize()() next we could """ functions = {"fetch": "fetch", "cleanimage": "cleanImage", "cleanbuild": "cleanBuild", "unpack": "unpack", "compile": "compile", "configure": "configure", "make": "make", "install": "install", "test": "unittest", "qmerge": "qmerge", "unmerge": "unmerge", "package": "createPackage", "createpatch": "createPatch", "geturls": "getUrls", "print-revision": "printSourceVersion", "print-files": "printFiles", "checkdigest": "checkDigest", - "dumpdeps": "dumpDependencies"} + "dumpdeps": "dumpDependencies", + "fetch-binary": "fetchBinary"} if command in functions: try: ok = getattr(self, functions[command])() except AttributeError as e: raise portage.PortageException( str( e ), self.category, self.package, e ) else: ok = EmergeDebug.error( "command %s not understood" % command ) return ok diff --git a/bin/Packager/PortablePackager.py b/bin/Packager/PortablePackager.py index dbd6bef26..f5826c531 100644 --- a/bin/Packager/PortablePackager.py +++ b/bin/Packager/PortablePackager.py @@ -1,45 +1,45 @@ # # copyright (c) 2011 Hannah von Reth # import EmergeDebug import EmergeHash import utils from .CollectionPackagerBase import * from .SevenZipPackager import * class PortablePackagerList( PackagerLists ): """ dummy name for PackagerLists """ class PortablePackager( CollectionPackagerBase, SevenZipPackager ): """ Packager for portal 7zip archives """ def __init__( self, whitelists=None, blacklists=None): CollectionPackagerBase.__init__( self, whitelists, blacklists ) SevenZipPackager.__init__(self) def createPortablePackage( self ): """create portable 7z package with digest files located in the manifest subdir""" if not self.packagerExe: EmergeDebug.die("could not find 7za in your path!") if not "setupname" in self.defines or not self.defines[ "setupname" ]: - self.defines[ "setupname" ] = self._archiveName("") + self.defines[ "setupname" ] = self.binaryArchiveName("") if not "srcdir" in self.defines or not self.defines[ "srcdir" ]: self.defines[ "srcdir" ] = self.imageDir() self._compress(self.defines[ "setupname" ], self.defines[ "srcdir" ], self.packageDestinationDir()) def createPackage( self ): """ create a package """ print("packaging using the PortablePackager") self.internalCreatePackage() self.createPortablePackage() EmergeHash.createDigestFiles(os.path.join(self.packageDestinationDir(), self.defines["setupname"])) return True diff --git a/bin/Packager/SevenZipPackager.py b/bin/Packager/SevenZipPackager.py index b647beaf2..d89db45d2 100644 --- a/bin/Packager/SevenZipPackager.py +++ b/bin/Packager/SevenZipPackager.py @@ -1,70 +1,56 @@ # # copyright (c) 2010 Ralf Habacker # # creates a 7z archive from the whole content of the package image # directory or optional from a sub directory of the image directory # This packager is in an experimental state - the implementation # and features may change in further versions # TODO: # - password support # - self extraction archives # # import EmergeDebug from Packager.PackagerBase import * class SevenZipPackager (PackagerBase): """Packager using the 7za command line tool from the dev-utils/7zip package""" def __init__( self, initialized = False ): if not initialized: PackagerBase.__init__( self ) fileName = "bin\\7za.exe" self.packagerExe = None for directory in [".", "dev-utils", "release", "debug"]: path = os.path.join(self.rootdir, directory, fileName ) if os.path.exists(path): self.packagerExe = path break if self.packagerExe: EmergeDebug.debug("using 7za from %s" % self.packagerExe, 2) - def _archiveName(self, pkgSuffix): - pkgVersion, _ = self.getPackageVersion() - if self.package.endswith( "-package" ): - shortPackage = self.package[ : -8 ] - else: - shortPackage = self.package - - return "%s-%s-%s%s%s.%s" % (shortPackage, compiler.architecture(), pkgVersion, compiler.getShortName(), pkgSuffix, emergeSettings.get("Packager", "7ZipArchiveType", "7z")) - def _compress(self, archiveName, sourceDir, destDir): utils.deleteFile(archiveName) cmd = "%s a -r %s %s/*" % (self.packagerExe, os.path.join(destDir, archiveName), sourceDir ) if not utils.system(cmd): EmergeDebug.die("while packaging. cmd: %s" % cmd) def createPackage(self): """create 7z package with digest files located in the manifest subdir""" if not self.packagerExe: EmergeDebug.die("could not find 7za in your path!") dstpath = self.packageDestinationDir() print(dstpath) - pkgSuffix = '' - if hasattr(self.subinfo.options.package, 'packageSuffix') and self.subinfo.options.package.packageSuffix: - pkgSuffix = self.subinfo.options.package.packageSuffix - - - self._compress(self._archiveName( pkgSuffix), self.imageDir(), dstpath) + self._compress(self.binaryArchiveName(), self.imageDir(), dstpath) if not self.subinfo.options.package.packSources: return True - self._compress(self._archiveName( "-src"), self.sourceDir(), dstpath) + self._compress(self.binaryArchiveName("-src"), self.sourceDir(), dstpath) return True diff --git a/bin/emerge.py b/bin/emerge.py index c43ab6d40..e482fc26a 100755 --- a/bin/emerge.py +++ b/bin/emerge.py @@ -1,492 +1,492 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # this will emerge some programs... # copyright: # Holger Schroeder # Patrick Spendrin # Hannah von Reth # The minimum python version for emerge please edit here # if you add code that changes this requirement import sys import EmergeDebug import EmergeTimer import time import datetime import traceback import argparse import collections import copy import compiler import portageSearch import InstallDB import portage import utils import threading from EmergeConfig import * def packageIsOutdated( category, package ): newest = portage.PortageInstance.getNewestVersion( category, package ) installed = InstallDB.installdb.getInstalledPackages( category, package ) for pack in installed: version = pack.getVersion( ) if newest != version: return True @utils.log def doExec( package, action, continueFlag = False ): with EmergeTimer.Timer("%s for %s" % ( action, package ), 1 ): EmergeDebug.info("Action: %s for %s" % (action, package)) ret = package.execute( action ) if not ret: EmergeDebug.warning("Action: %s for %s FAILED" % (action, package)) return ret or continueFlag def handlePackage( category, packageName, buildAction, continueFlag, skipUpToDateVcs ): with EmergeTimer.Timer("HandlePackage %s/%s" % (category, packageName), 3) as timer: EmergeDebug.debug_line() EmergeDebug.info("Handling package: %s, action: %s" % (packageName, buildAction)) success = True package = portage.getPackageInstance( category, packageName ) if package is None: raise portage.PortageException( "Package not found", category, packageName ) if buildAction in [ "all", "full-package", "update", "update-all" ]: success = success and doExec( package, "fetch", continueFlag ) skip = False if success and skipUpToDateVcs and package.subinfo.hasSvnTarget( ): revision = package.sourceVersion( ) for p in InstallDB.installdb.getInstalledPackages( category, packageName ): if p.getRevision( ) == revision: EmergeDebug.info("Skipping further actions, package is up-to-date") skip = True if not skip: success = success and doExec( package, "unpack", continueFlag ) success = success and doExec( package, "compile" ) success = success and doExec( package, "cleanimage" ) success = success and doExec( package, "install" ) if buildAction in [ "all", "update", "update-all" ]: success = success and doExec( package, "qmerge" ) if buildAction == "full-package": success = success and doExec( package, "package" ) success = success or continueFlag - elif buildAction in [ "fetch", "unpack", "configure", "compile", "make", "checkdigest", + elif buildAction in [ "fetch", "fetch-binary", "unpack", "configure", "compile", "make", "checkdigest", "dumpdeps", "test", "package", "unmerge", "cleanimage", "cleanbuild", "createpatch", "geturls", "print-revision", "print-files" ]: success = doExec( package, buildAction, continueFlag ) elif buildAction == "install": success = doExec( package, "cleanimage" ) success = success and doExec( package, "install", continueFlag ) elif buildAction == "qmerge": #success = doExec( package, "cleanimage" ) #success = success and doExec( package, "install") success = success and doExec( package, "qmerge" ) elif buildAction == "print-source-version": print( "%s-%s" % ( packageName, package.sourceVersion( ) ) ) success = True elif buildAction == "print-package-version": print( "%s-%s-%s" % ( packageName, compiler.getCompilerName( ), package.sourceVersion( ) ) ) success = True elif buildAction == "print-targets": portage.printTargets( category, packageName ) success = True else: success = EmergeDebug.error("could not understand this buildAction: %s" % buildAction) timer.stop() utils.notify( "Emerge %s %s" % (buildAction, "succeeded" if success else "failed"), "%s of %s/%s %s after %s" % ( buildAction, category, packageName, "succeeded" if success else "failed", timer), buildAction) return success def handleSinglePackage( packageName, action, args ): deplist = [ ] packageList = [ ] originalPackageList = [ ] categoryList = [ ] targetDict = dict( ) if action == "update-all": installedPackages = portage.PortageInstance.getInstallables( ) if portage.PortageInstance.isCategory( packageName ): EmergeDebug.debug("Updating installed packages from category " + packageName, 1) else: EmergeDebug.debug("Updating all installed packages", 1) packageList = [ ] for mainCategory, mainPackage in installedPackages: if portage.PortageInstance.isCategory( packageName ) and ( mainCategory != packageName ): continue if InstallDB.installdb.isInstalled( mainCategory, mainPackage, args.buildType ) \ and portage.isPackageUpdateable( mainCategory, mainPackage ): categoryList.append( mainCategory ) packageList.append( mainPackage ) EmergeDebug.debug("Will update packages: " + str(packageList), 1) elif args.list_file: listFileObject = open( args.list_file, 'r' ) for line in listFileObject: if line.strip( ).startswith( '#' ): continue try: cat, pac, tar, _ = line.split( ',' ) except: continue categoryList.append( cat ) packageList.append( pac ) originalPackageList.append( pac ) targetDict[ cat + "/" + pac ] = tar elif packageName: packageList, categoryList = portage.getPackagesCategories( packageName ) for entry in packageList: EmergeDebug.debug("Checking dependencies for: %s" % entry, 1) for mainCategory, entry in zip( categoryList, packageList ): deplist = portage.solveDependencies( mainCategory, entry, deplist, args.dependencyType, maxDepth = args.dependencydepth ) # no package found if len( deplist ) == 0: category = "" if not packageName.find( "/" ) == -1: (category, package) = packageName.split( "/" ) portageSearch.printSearch( category, packageName ) return False for item in deplist: item.enabled = args.ignoreAllInstalled if args.ignoreInstalled and item.category in categoryList and item.package in packageList or packageIsOutdated( item.category, item.package ): item.enabled = True if item.category + "/" + item.package in targetDict: item.target = targetDict[ item.category + "/" + item.package ] if args.target in list( portage.PortageInstance.getAllTargets( item.category, item.package ).keys( ) ): # if no target or a wrong one is defined, simply set the default target here item.target = args.target EmergeDebug.debug("dependency: %s" % item, 1) if not deplist: EmergeDebug.debug("", 1) EmergeDebug.debug_line(1) #for item in deplist: # cat = item[ 0 ] # pac = item[ 1 ] # ver = item[ 2 ] # if portage.isInstalled( cat, pac, ver, buildType) and updateAll and not portage.isPackageUpdateable( cat, pac, ver ): # print "remove:", cat, pac, ver # deplist.remove( item ) if action == "install-deps": # the first dependency is the package itself - ignore it # TODO: why are we our own dependency? del deplist[ 0 ] elif action == "update-direct-deps": for item in deplist: item.enabled = True deplist.reverse( ) # package[0] -> category # package[1] -> package # package[2] -> version info = deplist[ -1 ] if not portage.PortageInstance.isVirtualPackage( info.category, info.package ) and \ not action in [ "all", "install-deps"] and\ not args.list_file or\ action in ["print-targets"]:#not all commands should be executed on the deps if we are a virtual packages # if a buildAction is given, then do not try to build dependencies # and do the action although the package might already be installed. # This is still a bit problematic since packageName might not be a valid # package # for list files, we also want to handle fetching & packaging per package if not handlePackage( info.category, info.package, action, args.doContinue, args.update_fast ): return False else: if args.dumpDepsFile: dumpDepsFileObject = open( args.dumpDepsFile, 'w+' ) dumpDepsFileObject.write( "# dependency dump of package %s\n" % ( packageName ) ) for info in deplist: isVCSTarget = False if args.dumpDepsFile: dumpDepsFileObject.write( ",".join( [ info.category, info.package, info.target, "" ] ) + "\n" ) isLastPackage = info == deplist[ -1 ] if args.outDateVCS or (args.outDatePackage and isLastPackage): isVCSTarget = portage.PortageInstance.getUpdatableVCSTargets( info.category, info.package ) != [ ] isInstalled = InstallDB.installdb.isInstalled( info.category, info.package ) if args.list_file and action != "all": info.enabled = info.package in originalPackageList if ( isInstalled and not info.enabled ) and not ( isInstalled and (args.outDateVCS or ( args.outDatePackage and isLastPackage) ) and isVCSTarget ): if EmergeDebug.verbose() > 1 and info.package == packageName: EmergeDebug.warning("already installed %s/%s" % (info.category, info.package)) elif EmergeDebug.verbose() > 2 and not info.package == packageName: EmergeDebug.warning("already installed %s/%s" % (info.category, info.package)) else: # in case we only want to see which packages are still to be build, simply return the package name if args.probe: EmergeDebug.warning("pretending %s" % info) else: if action in [ "install-deps", "update-direct-deps" ]: action = "all" if not handlePackage( info.category, info.package, action, args.doContinue, args.update_fast ): EmergeDebug.error("fatal error: package %s/%s %s failed" % \ ( info.category, info.package, action )) return False EmergeDebug.new_line() return True class ActionHandler: class StoreTrueAction(argparse._StoreTrueAction): def __call__(self, parser, namespace, values, option_string=None): ActionHandler.StoreAction._addOrdered(namespace, self.dest, self.const) super().__call__(parser, namespace, values, option_string) class StoreAction(argparse._StoreAction): """http://stackoverflow.com/a/9028031""" def __call__(self, parser, namespace, values, option_string=None): ActionHandler.StoreAction._addOrdered(namespace, self.dest, values) super().__call__(parser, namespace, values, option_string) @staticmethod def _addOrdered(namespace, key, value): if not 'ordered_args' in namespace: setattr(namespace, 'ordered_args', collections.OrderedDict()) namespace.ordered_args[key] = value def __init__(self, parser): self.parser = parser self.actions = {} def _addAction(self, actionName, help = None, **kwargs): arg = self.parser.add_argument("--%s" % actionName, help = "[Action] %s" % (help if help else ""), **kwargs) self.actions[arg.dest] = actionName def addAction(self, actionName, **kwargs): self._addAction(actionName, action = ActionHandler.StoreTrueAction, **kwargs) def addActionWithArg(self, actionName, **kwargs): self._addAction(actionName, action = ActionHandler.StoreAction, **kwargs) def parseFinalAction(self, args, defaultAction): '''Returns the list of actions or [defaultAction]''' return [self.actions[x] for x in args.ordered_args.keys()] if hasattr(args, "ordered_args") else [defaultAction] def main( ): parser = argparse.ArgumentParser( prog = "emerge", description = "Emerge is a tool for building KDE-related software under Windows. emerge automates it, looks for the dependencies and fetches them automatically.\ Some options should be used with extreme caution since they will make your kde installation unusable in 999 out of 1000 cases.", epilog = """More information see the README or http://windows.kde.org/. Send feedback to .""" ) parser.add_argument( "-p", "--probe", action = "store_true", help = "probing: emerge will only look which files it has to build according to the list of installed files and according to the dependencies of the package." ) parser.add_argument( "--list-file", action = "store", help = "Build all packages from the csv file provided" ) _def = emergeSettings.get( "General", "EMERGE_OPTIONS", "" ) if _def == "": _def = [] else: _def = _def.split( ";" ) parser.add_argument( "--options", action = "append", default = _def, help = "Set emerge property from string . An example for is \"cmake.openIDE=1\" see options.py for more informations." ) parser.add_argument( "-z", "--outDateVCS", action = "store_true", help = "if packages from version control system sources are installed, it marks them as out of date and rebuilds them (tags are not marked as out of date)." ) parser.add_argument( "-sz", "--outDatePackage", action = "store_true", help = "similar to -z, only that it acts only on the last package, and works as normal on the rest." ) parser.add_argument( "-q", "--stayquiet", action = "store_true", dest = "stayQuiet", help = "quiet: there should be no output - The verbose level should be 0" ) parser.add_argument( "-t", "--buildtests", action = "store_true", dest = "buildTests", default = emergeSettings.getboolean( "Compile", "BuildTests", False ) ) parser.add_argument( "-c", "--continue", action = "store_true", dest = "doContinue" ) parser.add_argument( "--offline", action = "store_true", default = emergeSettings.getboolean( "General", "WorkOffline", False ), help = "do not try to connect to the internet: KDE packages will try to use an existing source tree and other packages would try to use existing packages in the download directory.\ If that doesn't work, the build will fail." ) parser.add_argument( "-f", "--force", action = "store_true", dest = "forced", default = emergeSettings.getboolean( "General", "EMERGE_FORCED", False ) ) parser.add_argument( "--buildtype", choices = [ "Release", "RelWithDebInfo", "MinSizeRel", "Debug" ], dest = "buildType", default = emergeSettings.get( "Compile", "BuildType", "RelWithDebInfo" ), help = "This will override the build type set in your kdesettings.ini." ) parser.add_argument( "-v", "--verbose", action = "count", default = int( emergeSettings.get( "EmergeDebug", "Verbose", "0" ) ), help = " verbose: increases the verbose level of emerge. Default is 1. verbose level 1 contains some notes from emerge, all output of cmake, make and other programs that are used.\ verbose level 2a dds an option VERBOSE=1 to make and emerge is more verbose highest level is verbose level 3." ) parser.add_argument( "--trace", action = "store", default = int( emergeSettings.get( "General", "EMERGE_TRACE", "0" ) ), type = int ) parser.add_argument( "-i", "--ignoreInstalled", action = "store_true", help = "ignore install: using this option will install a package over an existing install. This can be useful if you want to check some new code and your last build isn't that old." ) parser.add_argument( "-ia", "--ignoreAllInstalled", action = "store_true", help = "ignore all install: using this option will install all package over an existing install. This can be useful if you want to check some new code and your last build isn't that old." ) parser.add_argument( "--target", action = "store", help = "This will override the build of the default target. The default Target is marked with a star in the printout of --print-targets" ) parser.add_argument( "--search", action = "store_true", help = "This will search for a package or a description matching or similar to the search term." ) parser.add_argument( "--noclean", action = "store_true", default = emergeSettings.getboolean( "General", "EMERGE_NOCLEAN", False ), help = "this option will try to use an existing build directory. Please handle this option with care - it will possibly break if the directory isn't existing." ) parser.add_argument( "--clean", action = "store_false", dest = "noclean", help = "oposite of --noclean" ) parser.add_argument( "--patchlevel", action = "store", default = emergeSettings.get( "General", "EMERGE_PKGPATCHLVL", "" ), help = "This will add a patch level when used together with --package" ) parser.add_argument( "--log-dir", action = "store", default = emergeSettings.get( "General", "EMERGE_LOG_DIR", "" ), help = "This will log the build output to a logfile in LOG_DIR for each package. Logging information is appended to existing logs." ) parser.add_argument( "--dt", action = "store", choices = [ "both", "runtime", "buildtime" ], default = "both", dest = "dependencyType" ) parser.add_argument( "--update-fast", action = "store_true", help = "If the package is installed from svn/git and the revision did not change all steps after fetch are skipped" ) parser.add_argument( "-d", "--dependencydepth", action = "store", type = int, default = -1, help = "By default emerge resolves the whole dependency graph, this option limits the depth of the graph, so a value of 1 would mean only dependencies defined in that package" ) actionHandler = ActionHandler(parser) - for x in sorted( [ "fetch", "unpack", "configure", "compile", "make", + for x in sorted( [ "fetch", "fetch-binary", "unpack", "configure", "compile", "make", "install", "install-deps", "qmerge", "manifest", "package", "unmerge", "test", "checkdigest", "dumpdeps", "full-package", "cleanimage", "cleanbuild", "createpatch", "geturls"] ): actionHandler.addAction( x ) actionHandler.addAction( "update", help = "Update a single package" ) # read-only actions actionHandler.addAction( "print-source-version" ) actionHandler.addAction( "print-package-version" ) actionHandler.addAction( "print-targets", help = "This will show a list of available targets for the package" ) actionHandler.addAction( "print-installed", help = "This will show a list of all packages that are installed currently." ) actionHandler.addAction( "print-installable", help = "This will give you a list of packages that can be installed. Currently you don't need to enter the category and package: only the package will be enough." ) actionHandler.addAction( "print-revision", help = "Print the revision of the package and exit" ) actionHandler.addAction( "print-files", help = "Print the files installed by the package and exit" ) actionHandler.addActionWithArg( "search-file", help = "Print packages owning the file" ) # other actions actionHandler.addActionWithArg( "dump-deps-file", dest = "dumpDepsFile", help = "Output the dependencies of this package as a csv file suitable for emerge server." ) parser.add_argument( "packageNames", nargs = argparse.REMAINDER ) args = parser.parse_args( ) if args.stayQuiet: EmergeDebug.setVerbose(-1) elif args.verbose: EmergeDebug.setVerbose(args.verbose) emergeSettings.set( "General", "WorkOffline", args.offline ) emergeSettings.set( "General", "EMERGE_NOCLEAN", args.noclean ) emergeSettings.set( "General", "EMERGE_FORCED", args.forced ) emergeSettings.set( "Compile", "BuildTests", args.buildTests ) emergeSettings.set( "Compile", "BuildType", args.buildType ) emergeSettings.set( "PortageVersions", "DefaultTarget", args.target ) emergeSettings.set( "General", "EMERGE_OPTIONS", ";".join( args.options ) ) emergeSettings.set( "General", "EMERGE_LOG_DIR", args.log_dir ) emergeSettings.set( "General", "EMERGE_TRACE", args.trace ) emergeSettings.set( "General", "EMERGE_PKGPATCHLVL", args.patchlevel ) portage.PortageInstance.options = args.options if args.search: for package in args.packageNames: category = "" if not package.find( "/" ) == -1: (category, package) = package.split( "/" ) portageSearch.printSearch( category, package ) return True for action in actionHandler.parseFinalAction(args, "all"): tempArgs = copy.deepcopy(args) if action in [ "install-deps", "update", "update-all", "package" ] or tempArgs.update_fast: tempArgs.ignoreInstalled = True if action in [ "update", "update-all" ]: tempArgs.noclean = True EmergeDebug.debug("buildAction: %s" % action) EmergeDebug.debug("doPretend: %s" % tempArgs.probe, 1) EmergeDebug.debug("packageName: %s" % tempArgs.packageNames) EmergeDebug.debug("buildType: %s" % tempArgs.buildType) EmergeDebug.debug("buildTests: %s" % tempArgs.buildTests) EmergeDebug.debug("verbose: %d" % EmergeDebug.verbose(), 1) EmergeDebug.debug("trace: %s" % tempArgs.trace, 1) EmergeDebug.debug("KDEROOT: %s" % EmergeStandardDirs.emergeRoot(), 1) EmergeDebug.debug_line() if action == "print-installed": InstallDB.printInstalled( ) elif action == "print-installable": portage.printInstallables( ) elif action == "search-file": portage.printPackagesForFileSearch(tempArgs.search_file) elif tempArgs.list_file: handleSinglePackage( "", action, tempArgs ) else: for packageName in tempArgs.packageNames: if not handleSinglePackage( packageName, action, tempArgs ): return False return True if __name__ == '__main__': success= True with EmergeTimer.Timer("Emerge", 0): try: doUpdateTitle = True def updateTitle( startTime, title ): while ( doUpdateTitle ): delta = datetime.datetime.now( ) - startTime utils.OsUtils.setConsoleTitle( "emerge %s %s" % (title, delta) ) time.sleep( 1 ) tittleThread = threading.Thread( target = updateTitle, args = (datetime.datetime.now( ), " ".join( sys.argv[ 1: ] ),) ) tittleThread.setDaemon( True ) tittleThread.start( ) success = main() except KeyboardInterrupt: pass except portage.PortageException as e: if e.exception: EmergeDebug.debug(e.exception, 0) traceback.print_tb( e.exception.__traceback__) EmergeDebug.error(e) except Exception as e: print( e ) traceback.print_tb( e.__traceback__ ) finally: doUpdateTitle = False if emergeSettings.getboolean( "EmergeDebug", "DumpSettings", False ): emergeSettings.dump( ) if not success: exit( 1 ) diff --git a/bin/utils.py b/bin/utils.py index 9e8cf12ad..bc923123e 100644 --- a/bin/utils.py +++ b/bin/utils.py @@ -1,768 +1,768 @@ # -*- coding: utf-8 -*- """@brief utilities this file contains some helper functions for emerge """ # copyright: # Holger Schroeder # Patrick Spendrin # Ralf Habacker import configparser import datetime import ftplib import http.client import inspect import shutil import tarfile import urllib.error import urllib.parse import urllib.request import zipfile from operator import itemgetter import EmergeDebug import EmergeHash import Notifier.NotificationLoader from EmergeConfig import * from EmergeOS.osutils import OsUtils # TODO: Rename class UtilsCache(): _NIGTHLY_URLS = dict() @staticmethod def findApplication(app) -> str: appLocation = shutil.which(app) if not appLocation: EmergeDebug.warning("Emerge was unable to locate: %s" % app) return None return appLocation @staticmethod def getNightlyVersionsFromUrl(url, pattern, timeout = 10) -> [str]: """ Returns a list of possible version number matching the regular expression in pattern. :param url: The url to look for the nightly builds. :param pattern: A regular expression to match the version. :param timeout: :return: A list of matching strings or [None] """ if emergeSettings.getboolean("General", "WorkOffline"): EmergeDebug.info("Nightly builds unavailable for %s in offline mode." % url) return [None] if url in UtilsCache._NIGTHLY_URLS: return UtilsCache._NIGTHLY_URLS[url] else: try: with urllib.request.urlopen(url, timeout = timeout) as fh: data = str(fh.read(), "UTF-8") vers = re.findall( pattern , data) if not vers: print(data) raise Exception("Pattern %s does not match." % pattern) UtilsCache._NIGTHLY_URLS[url] = vers return vers except Exception as e: EmergeDebug.warning("Nightly builds unavailable for %s: %s" % (url, e)) return [None] def abstract(): caller = inspect.getouterframes(inspect.currentframe())[1][3] raise NotImplementedError(caller + ' must be implemented in subclass') def getCallerFilename(): """ returns the file name of the """ filename = None try: frame=inspect.currentframe() count = 2 while count > 0 and frame: frame = frame.f_back # python 3.3 includes unnecessary importlib frames, skip them if frame and frame.f_code.co_filename != '': count -= 1 finally: if frame: filename = frame.f_code.co_filename del frame return filename ### fetch functions def getFiles( urls, destdir, suffix='' , filenames = ''): """download files from 'url' into 'destdir'""" EmergeDebug.debug("getfiles called. urls: %s, filenames: %s, suffix: %s" % (urls, filenames, suffix), 1) # make sure distfiles dir exists if ( not os.path.exists( destdir ) ): os.makedirs( destdir ) if type(urls) == list: urlList = urls else: urlList = urls.split() if filenames == '': filenames = [ os.path.basename(x) for x in urlList ] if type(filenames) == list: filenameList = filenames else: filenameList = filenames.split() dlist = list(zip( urlList , filenameList )) for url,filename in dlist: if ( not getFile( url + suffix, destdir , filename ) ): return False return True -def getFile( url, destdir , filename='' ): +def getFile( url, destdir , filename='' ) -> bool: """download file from 'url' into 'destdir'""" EmergeDebug.debug("getFile called. url: %s" % url, 1) if url == "": EmergeDebug.error("fetch: no url given") return False if UtilsCache.findApplication("wget"): return wgetFile( url, destdir , filename ) if not filename: _, _, path, _, _, _ = urllib.parse.urlparse( url ) filename = os.path.basename( path ) if os.path.exists(os.path.join( destdir, filename )): return True width, _ = shutil.get_terminal_size((80,20)) def dlProgress(count, blockSize, totalSize): percent = int(count * blockSize * 100 / totalSize) times = int((width - 20)/100 * percent) sys.stdout.write(("\r%s%3d%%" % ("#" * times, percent))) sys.stdout.flush() urllib.request.urlretrieve(url, filename = os.path.join( destdir, filename ), reporthook= dlProgress if EmergeDebug.verbose() >= 0 else None ) if EmergeDebug.verbose()>=0: sys.stdout.write("\n") sys.stdout.flush() return True def wgetFile( url, destdir, filename=''): """download file with wget from 'url' into 'destdir', if filename is given to the file specified""" command = "\"%s\" -c -t 10" % UtilsCache.findApplication("wget") if EmergeDebug.Verbose().level() < 1: command += " -q --show-progress" if emergeSettings.getboolean("General", "EMERGE_NO_PASSIVE_FTP", False ): command += " --no-passive-ftp " if(filename ==''): command += " -P %s" % destdir else: command += " -O %s" % os.path.join( destdir, filename ) command += " %s" % url EmergeDebug.debug("wgetfile called", 1) ret = system( command ) EmergeDebug.debug("wget ret: %s" % ret, 2) return ret ### unpack functions def unpackFiles( downloaddir, filenames, workdir ): """unpack (multiple) files specified by 'filenames' from 'downloaddir' into 'workdir'""" for filename in filenames: if ( not unpackFile( downloaddir, filename, workdir ) ): return False return True def unpackFile( downloaddir, filename, workdir ): """unpack file specified by 'filename' from 'downloaddir' into 'workdir'""" EmergeDebug.debug("unpacking this file: %s" % filename) ( shortname, ext ) = os.path.splitext( filename ) if re.match( "(.*\.tar.*$|.*\.tgz$)", filename ): shutil.unpack_archive(os.path.join(downloaddir, filename),workdir) elif ext == "": return True return un7zip( os.path.join( downloaddir, filename ), workdir, ext ) def un7zip( fileName, destdir, flag = None ): command = os.path.join(EmergeStandardDirs.emergeRoot(), "emerge", "bin", "emerge_7za") command += " x -r -y -o%s %s" % ( destdir, fileName ) if flag == ".7z": # Actually this is not needed for a normal archive. # But git is an exe file renamed to 7z and we need to specify the type. # Yes it is an ugly hack. command += " -t7z" command += " -bsp1" if EmergeDebug.verbose() <= 1: command += " -bso0" return system( command ) def system(cmd, **kw ): """execute cmd in a shell. All keywords are passed to Popen. stdout and stderr might be changed depending on the chosen logging options.""" kw['shell'] = True return systemWithoutShell(cmd, **kw) def systemWithoutShell(cmd, **kw): """execute cmd. All keywords are passed to Popen. stdout and stderr might be changed depending on the chosen logging options.""" EmergeDebug.debug("executing command: %s" % cmd, 1) if EmergeDebug.verbose() == -1 and not 'stdout' in kw and not 'stderr' in kw: kw['stdout'] = kw['stderr'] = subprocess.DEVNULL return subprocess.call(cmd, **kw) == 0 def getFileListFromDirectory( imagedir ): """ create a file list containing hashes """ ret = [] myimagedir = imagedir if ( not imagedir.endswith( "\\" ) ): myimagedir = myimagedir + "\\" algorithm = EmergeHash.HashAlgorithm.SHA256 for root, _, files in os.walk( imagedir ): for fileName in files: ret.append( ( os.path.join( root, fileName ).replace( myimagedir, "" ), algorithm.stringPrefix() + EmergeHash.digestFile( os.path.join( root, fileName), algorithm) ) ) return ret def unmergeFileList(rootdir, fileList, forced=False): """ delete files in the fileList if has matches or forced is True """ for filename, filehash in fileList: fullPath = os.path.join(rootdir, os.path.normcase( filename)) if os.path.isfile(fullPath): algorithm = EmergeHash.HashAlgorithm.getAlgorithmFromPrefix(filehash) if not algorithm: currentHash = EmergeHash.digestFile(fullPath, EmergeHash.HashAlgorithm.MD5) else: currentHash = algorithm.stringPrefix() + EmergeHash.digestFile(fullPath, algorithm) if currentHash == filehash or filehash == "": OsUtils.rm(fullPath, True) else: if forced: EmergeDebug.warning("file %s has different hash: %s %s, deleting anyway" % \ (fullPath, currentHash, filehash )) OsUtils.rm(fullPath, True) else: EmergeDebug.warning("file %s has different hash: %s %s, run with option --force to delete it anyway" % \ (fullPath, currentHash, filehash )) elif not os.path.isdir(fullPath): EmergeDebug.warning("file %s does not exist" % fullPath) def mergeImageDirToRootDir( imagedir, rootdir , linkOnly = emergeSettings.getboolean("General", "UseHardlinks", False )): copyDir( imagedir, rootdir , linkOnly) def moveEntries( srcdir, destdir ): for entry in os.listdir( srcdir ): src = os.path.join( srcdir, entry ) dest = os.path.join( destdir, entry ) EmergeDebug.debug("move: %s -> %s" % (src, dest), 1) if( os.path.isfile( dest ) ): os.remove( dest ) if( os.path.isdir( dest ) ): continue os.rename( src, dest ) def cleanDirectory( directory ): EmergeDebug.debug("clean directory %s" % directory, 1) if ( os.path.exists( directory ) ): for root, dirs, files in os.walk( directory, topdown=False): for name in files: if not OsUtils.rm(os.path.join(root, name), True): EmergeDebug.die("couldn't delete file %s\n ( %s )" % (name, os.path.join(root, name))) for name in dirs: if not OsUtils.rmDir(os.path.join(root, name), True): EmergeDebug.die("couldn't delete directory %s\n( %s )" % (name, os.path.join(root, name))) else: os.makedirs( directory ) def getVCSType( url ): """ return the type of the vcs url """ if not url: return "" if isGitUrl( url ): return "git" elif url.find("://") == -1: return "svn" elif url.startswith("[hg]"): return "hg" elif url.find("svn:") >= 0 or url.find("https:") >= 0 or url.find("http:") >= 0: return "svn" ## \todo complete more cvs access schemes elif url.find("pserver:") >= 0: return "cvs" else: return "" def isGitUrl( Url ): """ this function returns true, if the Url given as parameter is a git url: it either starts with git:// or the first part before the first '|' ends with .git or if the url starts with the token [git] """ if Url.startswith('git://'): return True # split away branch and tags splitUrl = Url.split('|') if splitUrl[0].endswith(".git"): return True if Url.startswith("[git]"): return True return False def splitVCSUrl( Url ): """ this function splits up an url provided by Url into the server name, the path, a branch or tag; it will return a list with 3 strings according to the following scheme: git://servername/path.git|4.5branch|v4.5.1 will result in ['git://servername:path.git', '4.5branch', 'v4.5.1'] This also works for all other dvcs""" splitUrl = Url.split('|') if len(splitUrl) < 3: c = [x for x in splitUrl] for dummy in range(3 - len(splitUrl)): c.append('') else: c = splitUrl[0:3] return c def replaceVCSUrl( Url ): """ this function should be used to replace the url of a server this comes in useful if you e.g. need to switch the server url for a push url on gitorious.org """ configfile = os.path.join(EmergeStandardDirs.etcPortageDir(), "..", "emergehosts.conf" ) replacedict = dict() # FIXME handle svn/git usernames and settings with a distinct naming #todo WTF if ( ("General", "KDESVNUSERNAME") in emergeSettings and emergeSettings.get("General", "KDESVNUSERNAME") != "username" ) : replacedict[ "git://git.kde.org/" ] = "git@git.kde.org:" if os.path.exists( configfile ): config = configparser.ConfigParser() config.read( configfile ) # add the default KDE stuff if the KDE username is set. for section in config.sections(): host = config.get( section, "host" ) replace = config.get( section, "replace" ) replacedict[ host ] = replace for host in list(replacedict.keys()): if not Url.find( host ) == -1: Url = Url.replace( host, replacedict[ host ] ) break return Url def createImportLibs( dll_name, basepath ): """creating the import libraries for the other compiler(if ANSI-C libs)""" dst = os.path.join( basepath, "lib" ) if( not os.path.exists( dst ) ): os.mkdir( dst ) # check whether the required binary tools exist HAVE_GENDEF = UtilsCache.findApplication( "gendef" ) is not None USE_GENDEF = HAVE_GENDEF HAVE_LIB = UtilsCache.findApplication( "lib" ) is not None HAVE_DLLTOOL = UtilsCache.findApplication( "dlltool" ) is not None if EmergeDebug.verbose() > 1: print("gendef found:", HAVE_GENDEF) print("gendef used:", USE_GENDEF) print("lib found:", HAVE_LIB) print("dlltool found:", HAVE_DLLTOOL) dllpath = os.path.join( basepath, "bin", "%s.dll" % dll_name ) defpath = os.path.join( basepath, "lib", "%s.def" % dll_name ) exppath = os.path.join( basepath, "lib", "%s.exp" % dll_name ) imppath = os.path.join( basepath, "lib", "%s.lib" % dll_name ) gccpath = os.path.join( basepath, "lib", "%s.dll.a" % dll_name ) if not HAVE_GENDEF and os.path.exists( defpath ): HAVE_GENDEF = True USE_GENDEF = False if not HAVE_GENDEF: EmergeDebug.warning("system does not have gendef.exe") return False if not HAVE_LIB and not os.path.isfile( imppath ): EmergeDebug.warning("system does not have lib.exe (from msvc)") if not HAVE_DLLTOOL and not os.path.isfile( gccpath ): EmergeDebug.warning("system does not have dlltool.exe") # create .def if USE_GENDEF: cmd = "gendef - %s -a > %s " % ( dllpath, defpath ) system( cmd ) if( HAVE_LIB and not os.path.isfile( imppath ) ): # create .lib cmd = "lib /machine:x86 /def:%s /out:%s" % ( defpath, imppath ) system( cmd ) if( HAVE_DLLTOOL and not os.path.isfile( gccpath ) ): # create .dll.a cmd = "dlltool -d %s -l %s -k" % ( defpath, gccpath ) system( cmd ) if os.path.exists( defpath ): os.remove( defpath ) if os.path.exists( exppath ): os.remove( exppath ) return True def toMSysPath( path ): path = path.replace( '\\', '/' ) if ( len(path) > 1 and path[1] == ':' ): path = '/' + path[0].lower() + '/' + path[3:] return path def cleanPackageName( basename, packagename ): return os.path.basename( basename ).replace( packagename + "-", "" ).replace( ".py", "" ) def createDir(path): """Recursive directory creation function. Makes all intermediate-level directories needed to contain the leaf directory""" if not os.path.exists( path ): EmergeDebug.debug("creating directory %s " % (path), 2) os.makedirs( path ) return True def copyFile(src, dest,linkOnly = emergeSettings.getboolean("General", "UseHardlinks", False)): """ copy file from src to dest""" EmergeDebug.debug("copy file from %s to %s" % (src, dest), 2) destDir = os.path.dirname( dest ) if not os.path.exists( destDir ): os.makedirs( destDir ) if os.path.exists( dest ): EmergeDebug.warning("Overriding %s" % dest) OsUtils.rm( dest, True ) if linkOnly: try: os.link( src , dest ) return True except: EmergeDebug.warning("Failed to create hardlink %s for %s" % (dest, src)) shutil.copy(src,dest) return True def copyDir( srcdir, destdir,linkOnly = emergeSettings.getboolean("General", "UseHardlinks", False ) ): """ copy directory from srcdir to destdir """ EmergeDebug.debug("copyDir called. srcdir: %s, destdir: %s" % (srcdir, destdir), 2) if ( not srcdir.endswith( os.path.sep ) ): srcdir += os.path.sep if ( not destdir.endswith( os.path.sep ) ): destdir += os.path.sep for root, _, files in os.walk( srcdir ): # do not copy files under .svn directories, because they are write-protected # and the they cannot easily be deleted... if ( root.find( ".svn" ) == -1 ): tmpdir = root.replace( srcdir, destdir ) if not os.path.exists( tmpdir ): os.makedirs( tmpdir ) for fileName in files: copyFile(os.path.join( root, fileName ),os.path.join( tmpdir, fileName ), linkOnly) EmergeDebug.debug("copy %s to %s" % (os.path.join(root, fileName), os.path.join(tmpdir, fileName)), 2) return True def mergeTree(srcdir, destdir): """ copy directory from @p srcdir to @p destdir If a directory in @p destdir exists, just write into it """ fileList = os.listdir(srcdir) for i in fileList: src = os.path.join(srcdir, i) dest = os.path.join(destdir, i) if os.path.exists(dest): if os.path.isdir(dest): mergeTree(src, dest) continue else: os.remove(dest) shutil.move(src, destdir) # Cleanup (only removing empty folders) os.rmdir(srcdir) def moveDir( srcdir, destdir ): """ move directory from srcdir to destdir """ EmergeDebug.debug("moveDir called. srcdir: %s, destdir: %s" % (srcdir, destdir), 1) try: shutil.move( srcdir, destdir ) except Exception as e: EmergeDebug.warning(e) return False return True def rmtree( directory ): """ recursively delete directory """ EmergeDebug.debug("rmtree called. directory: %s" % (directory), 2) shutil.rmtree ( directory, True ) # ignore errors def moveFile(src, dest): """move file from src to dest""" EmergeDebug.debug("move file from %s to %s" % (src, dest), 2) shutil.move( src, dest ) return True def deleteFile(fileName): """delete file """ if not os.path.exists( fileName ): return False EmergeDebug.debug("delete file %s " % (fileName), 2) os.remove( fileName ) return True def findFiles( directory, pattern=None, fileNames=None): """find files recursivly""" if fileNames == None: fileNames = [] pattern = pattern.lower() for entry in os.listdir(directory): if entry.find(".svn") > -1 or entry.find(".bak") > -1: continue fileName = os.path.join(directory, entry) if os.path.isdir(fileName): findFiles(fileName, pattern, fileNames) elif os.path.isfile(fileName) and pattern == None or entry.lower().find(pattern) > -1: fileNames.append(fileName) return fileNames def putenv(name, value): """set environment variable""" EmergeDebug.debug("set environment variable -- set %s=%s" % (name, value), 2) os.putenv( name, value ) return True def applyPatch(sourceDir, f, patchLevel='0'): """apply single patch""" cmd = 'patch -d "%s" -p%s -i "%s"' % (sourceDir, patchLevel, f) EmergeDebug.debug("applying %s" % cmd) result = system( cmd ) if not result: EmergeDebug.warning("applying %s failed!" % f) return result def log(fn): def inner(*args, **argv): logdir = emergeSettings.get( "General", "EMERGE_LOG_DIR", "" ) if logdir == "": return fn(*args, **argv) if os.path.isfile(logdir): EmergeDebug.die("EMERGE_LOG_DIR %s is a file" % logdir) if not os.path.exists(logdir): try: os.mkdir(logdir) except OSError: EmergeDebug.die("EMERGE_LOG_DIR %s can not be created" % logdir) logfile = "" for a in args: logfile += "%s-" % a logfile = logfile[:-1]#drop last - logfile = "%s.log" % logfile.replace("/","_").replace("\\","_") logfile = os.path.join(logdir, logfile) f = open(logfile, "at") try: old_out = sys.stdout old_err = sys.stderr sys.stdout = f sys.stderr = f return fn(*args, **argv) finally: sys.stdout = old_out sys.stderr = old_err f.close() return inner def getWinVer(): ''' Returns the Windows Version of the system returns "0" if the Version can not be determined ''' try: result = str(subprocess.Popen("cmd /C ver", stdout=subprocess.PIPE).communicate()[0],"windows-1252") except OSError: EmergeDebug.debug("Windows Version can not be determined", 1) return "0" version = re.search(r"\d+\.\d+\.\d+", result) if(version): return version.group(0) EmergeDebug.debug("Windows Version can not be determined", 1) return "0" def regQuery(key, value): ''' Query the registry key for value and return the result. ''' query = 'reg query "%s" /v "%s"' % (key, value) EmergeDebug.debug("Executing registry query %s " % query, 2) result = subprocess.Popen(query, stdout = subprocess.PIPE).communicate()[0] # Output of this command is either an error to stderr # or the key with the value in the next line reValue = re.compile(r"(\s*%s\s*REG_\w+\s*)(.*)" % value) match = reValue.search(str(result, 'windows-1252')) if match and match.group(2): return match.group(2).rstrip() return False def embedManifest(executable, manifest): ''' Embed a manifest to an executable using either the free kdewin manifest if it exists in dev-utils/bin or the one provided by the Microsoft Platform SDK if it is installed' ''' if not os.path.isfile(executable) or not os.path.isfile(manifest): # We die here because this is a problem with the portage files EmergeDebug.die("embedManifest %s or %s do not exist" % (executable, manifest)) EmergeDebug.debug("embedding ressource manifest %s into %s" % \ (manifest, executable), 2) mtExe = None mtExe = os.path.join(EmergeStandardDirs.emergeRoot(), "dev-utils", "bin", "mt.exe") if(not os.path.isfile(mtExe)): # If there is no free manifest tool installed on the system # try to fallback on the manifest tool provided by visual studio sdkdir = regQuery("HKLM\SOFTWARE\Microsoft\Microsoft SDKs\Windows", "CurrentInstallFolder") if not sdkdir: EmergeDebug.debug("embedManifest could not find the Registry Key" " for the Windows SDK", 2) else: mtExe = r'%s' % os.path.join(sdkdir, "Bin", "mt.exe") if not os.path.isfile(os.path.normpath(mtExe)): EmergeDebug.debug("embedManifest could not find a mt.exe in\n\t %s" % \ os.path.dirname(mtExe), 2) if os.path.isfile(mtExe): return system([mtExe, "-nologo", "-manifest", manifest, "-outputresource:%s;1" % executable]) else: return system(["mt", "-nologo", "-manifest", manifest, "-outputresource:%s;1" % executable]) def getscriptname(): if __name__ == '__main__': return sys.argv[ 0 ] else: return __name__ def prependPath(*parts): """put path in front of the PATH environment variable, if it is not there yet. The last part must be a non empty string, otherwise we do nothing""" if parts[-1]: fullPath = os.path.join(*parts) old = os.getenv("PATH").split(';') if old[0] != fullPath: EmergeDebug.debug("adding %s to system path" % fullPath, 2) old.insert(0, fullPath) putenv( "PATH", ";".join(old)) def notify(title,message,alertClass = None): EmergeDebug.info("%s: %s" % (title, message)) backends = emergeSettings.get( "General","EMERGE_USE_NOTIFY", "") if backends == "": return backends = Notifier.NotificationLoader.load(backends.split(";")) for backend in backends.values(): backend.notify(title,message,alertClass) def levenshtein(s1, s2): if len(s1) < len(s2): return levenshtein(s2, s1) if not s1: return len(s2) previous_row = range(len(s2) + 1) for i, c1 in enumerate(s1): current_row = [i + 1] for j, c2 in enumerate(s2): insertions = previous_row[j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer deletions = current_row[j] + 1 # than s2 substitutions = previous_row[j] + (c1 != c2) current_row.append(min(insertions, deletions, substitutions)) previous_row = current_row return previous_row[-1] #taken from https://bitbucket.org/pypa/setuptools/src/a3d16c5f7443ec6e5e4d8d4791682b56130b41b5/pkg_resources.py?at=default def parse_version(s): """Convert a version string to a chronologically-sortable key This is a rough cross between distutils' StrictVersion and LooseVersion; if you give it versions that would work with StrictVersion, then it behaves the same; otherwise it acts like a slightly-smarter LooseVersion. It is *possible* to create pathological version coding schemes that will fool this parser, but they should be very rare in practice. The returned value will be a tuple of strings. Numeric portions of the version are padded to 8 digits so they will compare numerically, but without relying on how numbers compare relative to strings. Dots are dropped, but dashes are retained. Trailing zeros between alpha segments or dashes are suppressed, so that e.g. "2.4.0" is considered the same as "2.4". Alphanumeric parts are lower-cased. The algorithm assumes that strings like "-" and any alpha string that alphabetically follows "final" represents a "patch level". So, "2.4-1" is assumed to be a branch or patch of "2.4", and therefore "2.4.1" is considered newer than "2.4-1", which in turn is newer than "2.4". Strings like "a", "b", "c", "alpha", "beta", "candidate" and so on (that come before "final" alphabetically) are assumed to be pre-release versions, so that the version "2.4" is considered newer than "2.4a1". Finally, to handle miscellaneous cases, the strings "pre", "preview", and "rc" are treated as if they were "c", i.e. as though they were release candidates, and therefore are not as new as a version string that does not contain them, and "dev" is replaced with an '@' so that it sorts lower than than any other pre-release tag. """ def _parse_version_parts(s): component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE) replace = {'pre':'c', 'preview':'c','-':'final-','rc':'c','dev':'@'}.get for part in component_re.split(s): part = replace(part,part) if not part or part=='.': continue if part[:1] in '0123456789': yield part.zfill(8) # pad for numeric comparison else: yield '*'+part yield '*final' # ensure that alpha/beta/candidate are before final parts = [] for part in _parse_version_parts(s.lower()): if part.startswith('*'): if part<'*final': # remove '-' before a prerelease tag while parts and parts[-1]=='*final-': parts.pop() # remove trailing zeros from each series of numeric parts while parts and parts[-1]=='00000000': parts.pop() parts.append(part) return tuple(parts) def createBat(fileName, command): with open(fileName, "wt+") as bat: bat.write("@echo off\r\n") bat.write(command) bat.write("\r\n") diff --git a/kdesettings.ini b/kdesettings.ini index cea90d5fc..09ab8d423 100644 --- a/kdesettings.ini +++ b/kdesettings.ini @@ -1,161 +1,163 @@ ## This is the settings file for use with powershell. ## Copy it to your emerge/../etc and call ". .\kdeenv.ps1" or "kdeenv.bat" in your emerge checkout. ## You can use cmake like variables for values in the same section. ## See Paths/${DOWNLOADDIR} [General] ## Here you set the compiler to be used. ## mingw4 - use the mingw gcc compiler (recommended) ## msvc2010, msvc2012, msvc2013 or msvc2015 - use the Microsoft Visual C++ compiler KDECompiler = mingw4 ## possible values x86 or x64 Architecture = x86 ## This option should be set to False if you use the msvc 201X Express Edition 64bit compiler ## in all other cases, simply keep this option commented out #Native=False ## This option can be used to enable a notification backend. ## As soon as the buildprocess of a project has finished a notification will be displayed. ## Possible Backends: ## Snarl http://snarl.fullphat.net/ ## Toaster Toaster will display a Windows 8 toast notification ## Snore https://github.com/Snorenotify/Snorenotify. Snore supports multiple backends. You need to install snore-send using emerge. ## Pushover https://pushover.net - Pushover is a service to receive instant push notifications on your phone or tablet from a variety of sources. ## If using pushover, you may also need to set EMERGE_PUSHOVER_APP_TOKEN (emerge will use a default Token if unset) and EMERGE_PUSHOVER_USER_KEY, which is your user key #EMERGE_USE_NOTIFY = Snarl;Toaster;Pushover #EMERGE_PUSHOVER_APP_TOKEN = aJU9PRUb6nGUUM2idyLfXdU8S5q18i #EMERGE_PUSHOVER_USER_KEY = ## Speed up the merging of packages by using hard links UseHardlinks = True [Paths] ## This is the location of your python installation. ## This value must be set. Python = C:\ProgramFiles\Python ## Some applications may need python 2.7 #Python27 = C:\python27 ## Here you change the download directory. ## If you want, so you can share the same download directory between ## mingw and msvc. ## The default value is emerge/../download #DownloadDir = C:\kde\download ## This option defines the location for git checkouts. ## The default value is emerge/../download/git #KDEGitDir = ${DOWNLOADDIR}\git ## This option defines the location for svn checkouts. ## The default value is emerge/../download/svn #KDESVNDir = ${DOWNLOADDIR}\svn ## This option defines the location where the ccache files are stored. ## The default location is KDEROOT/build/ccache #CCACHE_DIR = C:\CCACHE\kf5 [Compile] ## Whether to build tests (default: True) #BuildTests = False ## the buildtype of this installation ## Possible Values: ## Release ## RelWithDebInfo ## Debug ## MinSizeRel BuildType = RelWithDebInfo ## Whether to use ninja (default: False) UseNinja = True ## Whether to use ccache (only avalible with mingw compiler) #UseCCache = True ## This option can be used to override the default make program ## change the value to the path of the executable you want to use instead. MakeProgram = jom [ShortPath] ## substitute pathes by drives ## This option is needed to avoid path limit problems in case of long base pathes ## and compiling big packages like qt ## If you disable it do _not_ use any paths longer than 6 letters in the ## directory settings EMERGE_USE_SHORT_PATH = False ## each drive could be commented out to skip substution EMERGE_ROOT_DRIVE = r: EMERGE_GIT_DRIVE = q: #EMERGE_DOWNLOAD_DRIVE = t: [Portage] ## This adds the possibility to disable cretin packages ## For portage recipes configure options can be added by ## "if self.subinfo.options.isActive("binary/mysql-pkg"):" ## Ignores = dev-util/git;dev-util/msys;kdesupport/kdewin;win32libs/boost/boost-python Ignores = [PortageVersions] ## Override the default target if this version is available. #DefaultTarget = 5.0 ## Overide the default version for a package. ## For a normal package add category/package, like win32libs/libpng and set the Version ## For meta packages like Qt5 you can directly set the version for the whole package #Qt5 = 5.2.1 #KF5 = 5.2.0 #KDE = 4.89.0 #Boost = 1_55_0 #win32libs/libpng = 1.2.43 #binary/vlc = 3.0.0-git [Packager] ## The archive type for packages. ## Possible values are: zip, 7z ## Todo: rename 7ZipArchiveType = zip [EmergeDebug] ## If you want to have verbose output, uncomment the following option ## and set it to positive integer for verbose output and to 0 ## (or disable it) for normal output. Currently the highest verbosity level ## is 3 (equal to 'emerge -v -v -v'). level -1 equals 'emerge -q' ## Default is Verbose = 0 #Verbose = 1 ## Prints time spend on various emerge tasks MeasureTime = False ## Dump internal state of emergeSettings to kdesettings.ini.dump #DumpSettings = True [Environment] ## All values defined here will be populated to the environment #GIT_COMMITTER_EMAIL = foo@bar.com ## Set the ssh client for git and svn. GIT_SSH = plink SVN_SSH = plink [QtSDK] ## This is higly experimental and you will miss certain features like dbus or mysql support. ## Whether to use prebuild Qt binaries. Enabled = False ## The path to the Qt sdk. Path = D:\Qt ## The version of Qt. Version = 5.3 ## The compiler version, if you are not sure what to use, have a look into the derectory set in QtSDK/Path. ## The compiler must be of the same type as General/KDECOMPILER. ## If you are using mingw please make sure you have installed the mingw using the Qt installer. Compiler = mingw482_32 +[ContinuousIntegration] +BinaryUrl = "" [Version] EMERGE_SETTINGS_VERSION = 2