diff --git a/bin/dependencies.py b/bin/dependencies.py index 7e6339792..86436f4da 100644 --- a/bin/dependencies.py +++ b/bin/dependencies.py @@ -1,637 +1,629 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright (c) 2010, Intevation GmbH # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of Intevation GmbH nor the # names of its contributors may be used to endorse or promote products # derived from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # """Tool to generate Graphviz DOT file of the package dependencies for a given emerge package. Doubles as a library for explict dependency graph traverals. Usage: Precondition: Set the environment variables needed by emerge. $ python bin/dependencies.py -t > deps.dot with being the kind of output send to stdout. Possible values: 'dot', 'kwi' and 'xml'. Defaults to 'dot' The output is send to stdout. Piped into a file you may use dot to generate a graphical version. $ dot -T svg -o deps.svg deps.dot """ __author__ = "Sascha L. Teichmann " __license__ = "New-style BSD" import portage import sys import os import utils import graphviz import xml2conf from argparse import ArgumentParser # pylint: disable=F0401 # this is for using pylint on Ubuntu which has still python2.6 # and no module argparse yet from string import Template # pylint: disable=W0402 OUTPUT_DOT = 0 OUTPUT_XML = 1 OUTPUT_KWI = 2 class Visitor(object): """Visitor applied to a node in the dependency graph during traversal """ CONTINUE_CHILDREN = 1 IGNORE_CHILDREN = 2 def beforeChildren(self, dummyNode, dummyContext): """Called before the children of the node are visited.""" return Visitor.CONTINUE_CHILDREN def afterChildren(self, dummyNode, dummyContext): """Called after the children of the node are visited.""" return Visitor.CONTINUE_CHILDREN def nlSeparated(node): """Replace ':' by newlines to make target look better in dot output.""" return str(node).replace(":", r"\n") class GraphvizCreator(Visitor): """Visitor to create DOT files from dependency graphs.""" def afterChildren(self, node, context): visited, out, ranks = context if not node.children: max_depth = 666 else: max_depth = node.maxDepth() ranks.setdefault(max_depth, set()).add(node) for child in node.children: link = '"%s" -> "%s"' % ( nlSeparated(node), nlSeparated(child)) if link not in visited: visited.add(link) out.append(link) return Visitor.CONTINUE_CHILDREN def createOutput(self, tree): visited = set() out = [ 'digraph "dependencies" {', 'ranksep=2;', 'size="6,6";' ] ranks = {} tree.visit(self, (visited, out, ranks)) for v in ranks.values(): out.append("{ rank=same; ") for n in v: out.append('"%s";' % nlSeparated(n)) out.append("}") out.append("}") return "\n".join(out) class XMLCreator(Visitor): """Visitor to create an XML representation of the dependency graph.""" def __init__(self): Visitor.__init__(self) self.nodes_so_far = {} self.ignored = False def beforeChildren(self, node, out): if not isinstance(node, DependenciesNode): return Visitor.CONTINUE_CHILDREN node_name = str(node) node_id = self.nodes_so_far.get(node_name) if node_id is not None: out.append('' % node_id) self.ignored = True return Visitor.IGNORE_CHILDREN new_id = len(self.nodes_so_far) self.nodes_so_far[node_name] = new_id out.append('") self.ignored = True return Visitor.IGNORE_CHILDREN out.append(">") return Visitor.CONTINUE_CHILDREN def afterChildren(self, dummyNode, out): if self.ignored: self.ignored = False else: out.append("") return Visitor.CONTINUE_CHILDREN def createOutput(self, tree): out = ['\n'] out.append("") tree.visit(self, out) out.append("") return ''.join(out) class KDEWinCreator( Visitor ): """ A visitor to generate a kdewin-installer config file """ compiler = "vc100" compilerlist = [ "x64-mingw4", "x86-mingw4", "vc100" ] mode = "deps" cats = dict() cats["data"] = [] cats["KDE"] = [] cats["kdesupport"] = [] cats["win32libs"] = [] def cleanup( self ): self.cats["data"] = [] self.cats["KDE"] = [] self.cats["kdesupport"] = [] self.cats["win32libs"] = [] def collectMetaData( self, node, context ): visited = context[0] if str( node ) not in visited: visited.add( str( node ) ) metaData = node.metaData packageName = node.package if node.category == "virtual": return None if packageName.endswith( "-src" ) or packageName.endswith( "-pkg" ): packageName = packageName[ : -4 ] if "shortDescription" in metaData: if metaData["withCompiler"]: asterisk = "-*" else: asterisk = "" return "@pkgnotes " + packageName + asterisk + " " + metaData[ "shortDescription" ] return None def collectVirtualPackages( self, node, context ): visited = context[0] if str( node ) not in visited: visited.add( str( node ) ) if node.virtual or portage.PortageInstance.isVirtualPackage( node.category, node.package ): if node.category == "kde": return "@metapackage " + node.package + " " + " ".join( [ x.package for x in node.children if not x.package.startswith("lib") ] ) return None def collectPackagesForCategory( self, node, context ): visited = context[0] if str( node ) not in visited: visited.add( str( node ) ) if node.category in [ "kdesupport", "libs" ] and node not in self.cats[ "kdesupport" ]: self.cats[ "kdesupport" ].append( node ) return None elif node.category.startswith( "kde" ) and node.category != "kdesupport" \ or node.category in [ "extragear", "kdeapps" ] and node not in self.cats[ "KDE" ]: self.cats[ "KDE" ].append( node ) return None elif node.category == "data" and node not in self.cats[ "data" ]: self.cats[ "data" ].append( node ) return None elif node.category in [ "testing", "win32libs" ] and node not in self.cats[ "win32libs" ]: self.cats[ "win32libs" ].append( node ) return None return None def __getNodeDependencies( self, node ): if node.metaData["withCompiler"]: depString = " runtime-" + self.compiler else: depString = "" for child in node.children: childName = child.package if child.category == "virtual": depString += self.__getNodeDependencies( child ) else: if childName.endswith( "-src" ) or childName.endswith( "-pkg" ): childName = childName[ :-4 ] depString += " " + childName if child.metaData["withCompiler"]: depString += "-" + self.compiler return depString def writeDependencyLine( self, node, context ): visited = context[0] packageName = node.package packageCategory = node.category if packageCategory == "virtual": return None if packageName.endswith( "-src" ) or packageName.endswith( "-pkg" ): packageName = packageName[ : -4 ] if "win32libs" in packageCategory: packageCategory = "win32libs" depString = "@deps " + packageName if node.metaData["withCompiler"]: depString += "-" + self.compiler deps = self.__getNodeDependencies( node ) depString += deps if str( node ) not in visited: # and len( node.children ) > 0: # since runtime packages are not included, use a hack here visited.add( str( node ) ) if len(deps) == 0: return None if not node.metaData["withCompiler"] and not node.dependencyVisited: node.dependencyVisited = True return depString elif not node.metaData["withCompiler"]: return None return depString return None def afterChildren( self, node, context ): out = context[1] if self.mode == "deps": result = self.writeDependencyLine( node, context ) elif self.mode == "cats": result = self.collectPackagesForCategory( node, context ) elif self.mode == "meta": result = self.collectMetaData( node, context ) elif self.mode == "virtual": result = self.collectVirtualPackages( node, context ) if result: out.append( result ) return Visitor.CONTINUE_CHILDREN def __dumpCategories( self, out ): out.append( "; to which category packages belong to" ) for _cat in self.cats: _str = "@categorypackages " + _cat num = 0 for _node in self.cats[ _cat ]: _packageName = _node.package if _packageName.endswith( "-src" ) or _packageName.endswith( "-pkg" ): _packageName = _packageName[:-4] if _node.metaData["withCompiler"]: _str += " " + _packageName + "-" + self.compiler num += 1 else: if not _node.categoryVisited: _node.categoryVisited = True _str += " " + _packageName num += 1 if num > 10: num = 0 out.append( _str ) _str = "@categorypackages " + _cat if num > 0: out.append( _str ) out.append( ";" ) def createOutput( self, tree ): out = [] tmpl_conf_path = os.path.join( os.path.dirname(__file__), "config.txt.template" ) with open(tmpl_conf_path) as f: template = Template(f.read()) visited = set() ranks = {} self.mode = "cats" for self.compiler in self.compilerlist: visited = set() tree.visit( self, ( visited, out, ranks ) ) self.__dumpCategories( out ) self.cleanup() _cat = "\n".join( out ) out = [] self.mode = "deps" for self.compiler in self.compilerlist: visited = set() out.append( "; package dependencies for compiler " + self.compiler ) tree.visit( self, ( visited, out, ranks ) ) out.append( ";" ) _dep = "\n".join( out ) out = [] visited = set() ranks = {} self.mode = "meta" tree.visit( self, ( visited, out, ranks ) ) _meta = "\n".join( out ) out = [] visited = set() ranks = {} self.mode = "virtual" tree.visit( self, ( visited, out, ranks ) ) _metapackages = "\n".join( out ) return template.safe_substitute( { 'metapackages': _metapackages, 'categorypackages': _cat, 'dependencies': _dep, 'pkgnotes': _meta } ) class DependenciesNode(object): """A node in the dependency graph.""" def __init__(self, category, package, version, tag = "1", children = None): if children is None: children = [] self.category = category self.package = package self.version = version self.tag = tag self.children = children self.parents = [] self.metaData = {'withCompiler': True} self.virtual = False self.categoryVisited = False self.dependencyVisited = False def __str__(self): return "%s:%s:%s:%s" % ( self.category, self.package, self.version, self.tag) def visit(self, visitor, context): """Apply a visitor to this node.""" if visitor.beforeChildren(self, context) == Visitor.CONTINUE_CHILDREN: for child in self.children: child.visit(visitor, context) visitor.afterChildren(self, context) def maxDepth(self): """Calculates the maximum depth of this node.""" if not self.parents: return 0 pdepth = -1 for p in self.parents: d = p.maxDepth() if d > pdepth: pdepth = d # TODO: use something like max(x.maxDepth() for x in self.parents) return pdepth + 1 class DependenciesTree(object): """A dependency tree. More a kind of DAG (directed acyclic graph).""" def __init__(self): self.roots = [] self.key2node = {} def __buildSubNodes(self, rootnode, converter): if rootnode.package not in converter.packageDepsList: return for deps in converter.packageDepsList[ rootnode.package ]: _cat, _pac = deps.split('/') if rootnode.category + "/" + _pac in list(converter.packageDepsList.keys()): _ver = rootnode.version _tag = rootnode.tag else: if portage.PortageInstance.isPackage(_cat, _pac): _ver = portage.PortageInstance.getNewestVersion(_cat, _pac) else: _ver = rootnode.version try: _tag = portage.PortageInstance.getDefaultTarget( _cat, _pac, _ver ) except ImportError: _tag = "1" subkey = "%s-%s-%s-%s" % (_cat, _pac, _ver, _tag) try: subnode = self.key2node[subkey] except KeyError: subnode = DependenciesNode(_cat, _pac, _ver, _tag) if _pac in list(converter.packageDescriptionList.keys()): subnode.metaData['shortDescription'] = converter.packageDescriptionList[_pac] if _pac == converter.moduleMetaName: subnode.virtual = True self.__buildSubNodes(subnode, converter) self.key2node[subkey] = subnode rootnode.children.append( subnode ) def buildVirtualNodes(self, category, package, version, tag, dep_type="both"): converter = xml2conf.Xml2Conf() converter.parseFile(os.path.join(portage.getDirname(category, package), package + "-package.xml")) key = "%s-%s-%s-%s" % (category, package, version, tag) try: node = self.key2node[key] return node except KeyError: pass rootnode = DependenciesNode(category, package, version, tag, []) rootnode.metaData = portage.PortageInstance.getMetaData(category, package, version) if package == converter.moduleMetaName: rootnode.virtual = True self.__buildSubNodes(rootnode, converter) return rootnode def addDependencies(self, category, package, version = "", dep_type="both"): """Add a new root dependency tree to this graph.""" pi = portage.PortageInstance - if portage.emergePlatform.isCrossCompilingEnabled(): - sp = pi.getCorrespondingSourcePackage(package) - if sp: - # we found such a package and we're allowed to replace it - category = sp[0] - package = sp[1] - version = pi.getNewestVersion(category, package) - if category == "": category = pi.getCategory(package) if version == "": version = pi.getNewestVersion(category, package) try: tag = pi.getDefaultTarget( category, package, version ) except ImportError: tag = "1" if not os.path.exists( os.path.join( portage.getDirname( category, package ), package + "-package.xml" ) ): node = self.buildDepNode( category, package, version, tag, dep_type ) else: node = self.buildVirtualNodes( category, package, version, tag, dep_type ) if not node in self.roots: self.roots.append(node) def buildDepNode(self, category, package, version, tag, dep_type="both"): """Recursive method to construct the nodes of the dependency tree.""" pi = portage.PortageInstance if portage.emergePlatform.isCrossCompilingEnabled(): sp = pi.getCorrespondingSourcePackage(package) if sp: # we found such a package and we're allowed to replace it category = sp[0] package = sp[1] version = pi.getNewestVersion(category, package) try: tag = pi.getDefaultTarget( category, package, version ) except ImportError: tag = "1" key = "%s-%s-%s-%s" % (category, package, version, tag) try: node = self.key2node[key] return node except KeyError: pass children = [] for t in portage.getDependencies(category, package, version, (dep_type=="runtime")): sub_node = self.buildDepNode(t[0], t[1], t[2], tag, dep_type) children.append(sub_node) node = DependenciesNode(category, package, version, tag, children) node.metaData = pi.getMetaData(category, package, version) for child in children: child.parents.append(node) self.key2node[key] = node return node def visit(self, visitor, context): """Apply a visitor to all parts of this graph.""" for root in self.roots: root.visit(visitor, context) def parseOptions(): usage = "%(prog)s [options] CATEGORY/PACKAGE\n" \ " or: %(prog)s -f packagelist.txt [packagelist2.txt ...]\n" \ " or: %(prog)s --file=packagelist.txt \n" parser = ArgumentParser(prog=sys.argv[0], usage=usage) parser.add_argument("-t", "--type", action = "store", default = OUTPUT_DOT, help="Change the output format type possible values: xml kwi, dot") parser.add_argument("-f", "--file", dest = "filenames", metavar = "FILENAME", nargs="+", help="add a filename for a packageList") parser.add_argument("-o", "--output", dest = "outputname", metavar = "FILENAME", help="the name of the output file") parser.add_argument("-d", "--depstyle", action = "store", default = "both", help="possible values: both, runtime") args, rest = parser.parse_known_args() return rest, args def parsePackageListFiles( filenames ): depList = [] for filename in filenames: with open(filename) as packagefile: for line in packagefile: if not line.startswith( '#' ): cat, pac, target, patchlvl = line.strip().split( ',' ) if pac.endswith( "-src" ): cat, pac = portage.PortageInstance.getCorrespondingBinaryPackage(pac) depList.append( ( cat, pac, target, patchlvl ) ) return depList def main(): rest, args = parseOptions() if args.type == "xml": output_type = OUTPUT_XML elif args.type == "kwi": output_type = OUTPUT_KWI else: output_type = OUTPUT_DOT depstyle = args.depstyle if not args.depstyle in ['both', 'runtime']: depstyle = "both" output = "" if hasattr(args, "filenames") and args.filenames != None: packageList = parsePackageListFiles( args.filenames ) output = dumpDependenciesForPackageList(packageList, output_type, depstyle) elif rest: output = dumpDependencies(rest[0], output_type, depstyle) else: utils.error("missing package list file or package/category") sys.exit(1) if hasattr(args, "outputname") and args.outputname: print("writing file ", args.outputname, os.path.dirname( args.outputname )) if os.path.dirname( args.outputname ) and not os.path.exists( os.path.dirname( args.outputname ) ): os.makedirs( os.path.dirname( args.outputname ) ) with open(args.outputname, "w") as f: f.write( output ) if output_type == OUTPUT_DOT: _graphviz = graphviz.GraphViz() if not _graphviz.runDot( args.outputname, args.outputname + '.pdf', 'pdf' ): exit( 1 ) # we don't want to open the output automatically, at least not always # _graphviz.openOutput() else: print(output) def createOutput(output_type, dep_tree): """return output for output_type""" if output_type == OUTPUT_XML: creator = XMLCreator() elif output_type == OUTPUT_DOT: creator = GraphvizCreator() elif output_type == OUTPUT_KWI: creator = KDEWinCreator() else: assert False, 'unknown output_type %d' % output_type return creator.createOutput(dep_tree) def dumpDependencies(category, output_type=OUTPUT_DOT, dep_type="both"): """without displaying debuginfo in generated output""" with utils.TemporaryVerbosity(0): packageList, categoryList = portage.getPackagesCategories(category) dep_tree = DependenciesTree() for _category, _package in zip(categoryList, packageList): dep_tree.addDependencies(_category, _package, dep_type=dep_type) return createOutput(output_type, dep_tree) def dumpDependenciesForPackageList(packageList, output_type=OUTPUT_DOT, dep_type="both"): """without displaying debuginfo in generated output""" with utils.TemporaryVerbosity(0): dep_tree = DependenciesTree() for category, package, dummyTarget, dummyPatchlevel in packageList: dep_tree.addDependencies(category, package, dep_type=dep_type) return createOutput(output_type, dep_tree) if __name__ == '__main__': main() diff --git a/bin/fetchlist.py b/bin/fetchlist.py deleted file mode 100644 index 81a05ed5c..000000000 --- a/bin/fetchlist.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import sys -import subprocess - -import portage -from packagelistparser import PackageListParser - -__doc__ = """this script produces a fetchlist for wget to synchronize the -win32libs packages from sourceforge by asking emerge with --geturls. -You must run this script for each compiler and add the fetchlist outputfile -file as a second parameter. New urls will get appended to the fetchlist, so -you must make sure that you start with an empty fetchlist file.""" - -if not len(sys.argv) == 3: - print("wrong number of arguments") - print() - print("syntax:") - print(sys.argv[0] + " packageListFile.txt outputfile") - print() - print(__doc__) - exit(1) - -parser = PackageListParser(sys.argv[1]) -_fetchListFile = open(sys.argv[2], "a+b") - -os.environ["EMERGE_PACKAGETYPES"] = "dbg,src" -for _cat, _pac, _ver, _patch in parser.getFullList(): - cat, pac = portage.PortageInstance.getCorrespondingBinaryPackage(_pac) - if cat == None or pac == None: - continue - - cmd = "emerge -q --geturls " + cat + "/" + pac - try: - output = subprocess.check_output(cmd.split(' '), shell=True) - except: - continue - print(output) - _fetchListFile.write(output) \ No newline at end of file diff --git a/bin/portage.py b/bin/portage.py index a1e01cf54..4d66d934d 100644 --- a/bin/portage.py +++ b/bin/portage.py @@ -1,859 +1,825 @@ ## @package portage # @brief contains portage tree related functions # @note this file should replace all other related portage related files import utils import builtins import imp import os import re import sys import portage_versions import emergePlatform import copy internalCategory = 'internal' ROOTDIR = os.getenv( "KDEROOT" ) modDict = dict() packageDict = dict() def __import__( module ): # pylint: disable=W0622 utils.debug( "module to import: %s" % module, 2 ) if not os.path.isfile( module ): try: return builtins.__import__( module ) except ImportError as e: utils.warning( 'import failed for module %s: %s' % (module, str(e)) ) return None else: sys.path.append( os.path.dirname( module ) ) modulename = os.path.basename( module ).replace('.py', '') suff_index = None for suff in imp.get_suffixes(): if suff[0] == ".py": suff_index = suff break if suff_index is None: utils.die("no .py suffix found") with open( module ) as fileHdl: try: return imp.load_module( modulename.replace('.', '_'), fileHdl, module, suff_index ) except ImportError as e: utils.warning( 'import failed for file %s: %s' % (module, e) ) return None class DependencyPackage(object): """ This class wraps each package and constructs the dependency tree original code is from dependencies.py, integration will come later... """ def __init__( self, category, name, version ): self.category = category self.name = name self.version = version self.runtimeChildren = [] self.buildChildren = [] self.__readChildren() def __eq__( self, other ): return self.category == other.category and self.name == other.name and self.version == other.version def __ne__( self, other ): return self.category != other.category or self.name != other.name or self.version != other.version def ident( self ): return [ self.category, self.name, self.version, PortageInstance.getDefaultTarget( self.category, self.name, self.version ) ] def __readChildren( self ): runtimeDependencies, buildDependencies = readChildren( self.category, self.name, self.version ) self.runtimeChildren = self.__readDependenciesForChildren( list(runtimeDependencies.keys()) ) self.buildChildren = self.__readDependenciesForChildren( list(buildDependencies.keys()) ) def __readDependenciesForChildren( self, deps ): children = [] if deps: for line in deps: ( category, package ) = line.split( "/" ) - if emergePlatform.isCrossCompilingEnabled(): - sp = PortageInstance.getCorrespondingSourcePackage( package ) - if sp: - # we found such a package and we're allowed to replace it - category = sp[ 0 ] - package = sp[ 1 ] - line = "%s/%s" % ( category, package ) - utils.debug( "category: %s, name: %s" % ( category, package ), 1 ) version = PortageInstance.getNewestVersion( category, package ) if not line in list(packageDict.keys()): p = DependencyPackage( category, package, version ) utils.debug( "adding package p %s/%s-%s" % ( category, package, version ), 1 ) packageDict[ line ] = p else: p = packageDict[ line ] children.append( p ) return children def getDependencies( self, depList=None, dep_type="both" ): """ returns all dependencies """ if depList is None: depList = [] if dep_type == "runtime": children = self.runtimeChildren elif dep_type == "buildtime": children = self.buildChildren else: children = self.runtimeChildren + self.buildChildren for p in children: if not p in depList: p.getDependencies( depList, dep_type ) #if self.category != internalCategory: if not self in depList: depList.append( self ) return depList def buildType(): """return currently selected build type""" return os.getenv( "EMERGE_BUILDTYPE" ) def rootDirectories(): # this function should return all currently set portage directories if os.getenv( "EMERGE_PORTAGE_ROOT" ): rootDirs = os.getenv( "EMERGE_PORTAGE_ROOT" ).split( ";" ) else: rootDirs = [] if len( rootDirs ) == 0: rootDirs = [ os.path.join( os.getenv( "KDEROOT" ), "emerge", "portage" ) ] return rootDirs def rootDir(): # this function should return the portage directory, either the first set # via the environment, or the default one # keep this function for compat reasons return rootDirectories()[0] def rootDirForCategory( category ): # this function should return the portage directory where it finds the # first occurance of a category or the default value for i in rootDirectories(): if category and os.path.exists( os.path.join( i, category ) ): return i # as a fall back return the default even if it might be wrong return os.path.join( os.getenv( "KDEROOT" ), "emerge", "portage" ) def rootDirForPackage( category, package ): # this function should return the portage directory where it finds the # first occurance of a package or the default value package, subpackage = getSubPackage( category, package ) if category and package: if subpackage: for i in rootDirectories(): if os.path.exists( os.path.join( i, category, package, subpackage ) ): return i else: for i in rootDirectories(): if os.path.exists( os.path.join( i, category, package ) ): return i # as a fall back return the default even if it might be wrong return os.path.join( os.getenv( "KDEROOT" ), "emerge", "portage" ) def getDirname( category, package ): """ return absolute pathname for a given category and package """ _package, _subpackage = getSubPackage( category, package ) if category and _package: if _subpackage: return os.path.join( rootDirForPackage( category, package ), category, _package, _subpackage ) else: return os.path.join( rootDirForPackage( category, package ), category, package ) else: return "" def getFilename( category, package, version ): """ return absolute filename for a given category, package and version """ return os.path.join( getDirname( category, package ), "%s-%s.py" % ( package, version ) ) def getCategoryPackageVersion( path ): utils.debug( "getCategoryPackageVersion: %s" % path, 2 ) head, fullFileName = os.path.split( path ) for rd in rootDirectories(): if head.startswith(rd): head = head.replace(rd + os.sep, "") break if len( head.split( os.sep ) ) == 3: category, _dummy, package = head.split( os.sep ) else: head, package = os.path.split( head ) head, category = os.path.split( head ) filename = os.path.splitext( fullFileName )[0] package, version = utils.packageSplit( filename ) utils.debug( "category: %s, package: %s, version: %s" % ( category, package, version ), 1 ) return [ category, package, version ] # TODO: why a list and not a tuple? def VCSDirs(): return [ '.svn', 'CVS', '.hg', '.git' ] class Portage(object): def __init__( self ): """ """ self.categories = {} self.subpackages = {} self.portages = {} def addPortageDir( self, directory ): """ adds the categories and packages of a portage directory """ if not os.path.exists( directory ): return categoryList = os.listdir( directory ) # remove vcs directories for vcsdir in VCSDirs(): if vcsdir in categoryList: categoryList.remove( vcsdir ) self.portages[ directory ] = [] for category in categoryList: if not os.path.isdir( os.path.join( directory, category ) ): continue self.portages[ directory ].append( category ) packageList = os.listdir( os.path.join( directory, category ) ) # remove vcs directories for vcsdir in VCSDirs(): if vcsdir in packageList: packageList.remove( vcsdir ) if not category in list(self.categories.keys()): self.categories[ category ] = [] for package in packageList: if not os.path.isdir( os.path.join( directory, category, package ) ): continue if not package in self.categories[ category ]: self.categories[ category ].append( package ) subPackageList = os.listdir( os.path.join( directory, category, package ) ) for subPackage in subPackageList: if not os.path.isdir( os.path.join( directory, category, package, subPackage ) ) or subPackage in VCSDirs(): continue if not subPackage in list(self.subpackages.keys()): self.subpackages[ subPackage ] = [] if not subPackage in self.categories[ category ]: self.categories[ category ].append( subPackage ) self.subpackages[ subPackage ].append( category + "/" + package ) def getCategory( self, package ): """ returns the category of this package """ utils.debug( "getCategory: %s" % package, 2 ) for cat in list(self.categories.keys()): if package in self.categories[ cat ]: utils.debug( "found: category %s for package %s" % ( cat, package ), 1 ) return cat return False def isCategory( self, category ): """ returns whether a certain category exists """ return category in list(self.categories.keys()) def isPackage( self, category, package ): """ returns whether a certain package exists within a category """ return package in self.categories[ category ] def isVirtualPackage( self, category, package ): """ check if that package is of VirtualPackageBase """ if not self.isPackage( category, package ): return False version = self.getNewestVersion( category, package ) mod = __import__( getFilename( category, package, version ) ) if hasattr( mod, 'Package' ): for baseClassObject in mod.Package.__bases__: if baseClassObject.__name__ == 'VirtualPackageBase': return True return False def getAllPackages( self, category ): """returns all packages of a category except those that are listed in a file 'dont_build.txt' in the category directory in case the category doesn't exist, nothing is returned""" if self.isCategory( category ): plist = copy.copy(self.categories[ category ]) if os.path.exists( os.path.join( rootDirForCategory( category ), category, "dont_build.txt" ) ): with open( os.path.join( rootDirForCategory( category ), category, "dont_build.txt" ), "r" ) as f: for line in f: try: plist.remove( line.strip() ) except ValueError: utils.warning( "couldn't remove package %s from category %s's package list" % ( line.strip(), category ) ) return plist else: return def getPackageInstance(self, category, package, buildtarget=None): """return instance of class Package from package file""" - if emergePlatform.isCrossCompilingEnabled(): - sp = self.getCorrespondingSourcePackage( package ) - if sp: - category = sp[0] - package = sp[1] - version = self.getNewestVersion( category, package ) fileName = getFilename( category, package, version ) module = __import__( fileName ) p = module.Package() p.setup(fileName, category, package, version, buildtarget) return p def getDefaultTarget( self, category, package, version ): """ returns the default package of a specified package """ utils.debug( "importing file %s" % getFilename( category, package, version ), 1 ) if not ( category and package and version ): return dict() mod = __import__( getFilename( category, package, version ) ) if hasattr( mod, 'subinfo' ): return mod.subinfo().defaultTarget else: return None def getMetaData( self, category, package, version ): """ returns all targets of a specified package """ utils.debug( "importing file %s" % getFilename( category, package, version ), 1 ) if not ( category and package and version ): return dict() mod = __import__( getFilename( category, package, version ) ) if hasattr( mod, 'subinfo' ): info = mod.subinfo() tmpdict = dict() if not info.categoryName == "": tmpdict['categoryName'] = info.categoryName if not info.shortDescription == "": tmpdict['shortDescription'] = info.shortDescription if not info.description == "": tmpdict['description'] = info.description if not info.homepage == "": tmpdict['homepage'] = info.homepage tmpdict['withCompiler'] = info.options.package.withCompiler utils.debug( tmpdict, 2 ) return tmpdict else: return {'withCompiler': True} def getAllTargets( self, category, package, version ): """ returns all targets of a specified package """ utils.debug( "importing file %s" % getFilename( category, package, version ), 1 ) if not ( category and package and version ): return dict() mod = __import__( getFilename( category, package, version ) ) if hasattr( mod, 'subinfo' ): info = mod.subinfo() tagDict = info.svnTargets tagDict.update( info.targets ) utils.debug( tagDict, 2 ) return tagDict else: return dict() def getAllVCSTargets( self, category, package, version ): """ returns all version control system targets of a specified package, excluding those which do contain tags """ utils.debug( "importing file %s" % getFilename( category, package, version ), 1 ) mod = __import__( getFilename( category, package, version ) ) if hasattr( mod, 'subinfo' ): info = mod.subinfo() tagDict = info.svnTargets for key in tagDict: utils.debug( '%s: %s' % ( key, tagDict[key] ), 2 ) return tagDict else: return dict() def getUpdatableVCSTargets( self, category, package, version ): """ check if the targets are tags or not """ targetDict = PortageInstance.getAllVCSTargets( category, package, version ) retList = [] for key in targetDict: url = targetDict[ key ] if url: sourceType = utils.getVCSType( url ) if sourceType == "svn": # for svn, ignore tags if not url.startswith( "tags/" ) and not "/tags/" in url: retList.append( key ) elif sourceType == "git": _, branch, tag = utils.splitVCSUrl( url ) if tag == "" and not branch.endswith("-patched"): retList.append( key ) elif not sourceType == "": # for all other vcs types, simply rebuild everything for now retList.append( key ) return retList def getNewestVersion( self, category, package ): """ returns the newest version of this category/package """ if( category == None ): utils.die( "Empty category for package '%s'" % package ) if not self.isCategory( category ): utils.die( "could not find category '%s'" % category ) if not self.isPackage( category, package ): utils.die( "could not find package '%s' in category '%s'" % ( package, category ) ) versions = [] for fileName in os.listdir( getDirname( category, package ) ): (shortname, ext) = os.path.splitext( fileName ) if ( ext != ".py" ): continue if ( shortname.startswith( package ) ): versions.append( shortname ) tmpver = "" for ver in versions: if ( tmpver == "" ): tmpver = ver else: ret = portage_versions.pkgcmp(portage_versions.pkgsplit(ver), \ portage_versions.pkgsplit(tmpver)) if ( ret == 1 ): tmpver = ver ret = utils.packageSplit( tmpver ) #print "ret:", ret return ret[ 1 ] def getInstallables( self ): """ get all the packages that are within the portage directory """ instList = list() for category in list(self.categories.keys()): for package in self.categories[ category ]: for script in os.listdir( getDirname( category, package ) ): if script.endswith( '.py' ): version = script.replace('.py', '').replace(package + '-', '') instList.append([category, package, version]) return instList - def getCorrespondingSourcePackage( self, package ): - category = self.getCategory( package + "-src" ) - if category: - # we found a corresponding package - utils.debug( "replacing package %s with its source package" % ( package ), 1 ) - return [ category, package + "-src" ] - else: - return False - - def getCorrespondingBinaryPackage( self, package ): - if not package.endswith( "-src" ): - return [ None, None ] - category = self.getCategory( package[ :-4 ] ) - if category: - # we found a corresponding package - utils.debug( "replacing package %s with its binary package" % ( package ), 1 ) - return [ category, package[ :-4 ] ] - else: - return [ None, None ] - # when importing this, this static Object should get added PortageInstance = Portage() for _dir in rootDirectories(): PortageInstance.addPortageDir( _dir ) def getSubPackage( category, package ): """ returns package and subpackage names """ """ in case no subpackage could be found, None is returned """ if package in PortageInstance.subpackages: for entry in PortageInstance.subpackages[ package ]: cat, pac = entry.split("/") if cat == category: return pac, package return package, None def findPossibleTargets( category, package, version, buildtype=''): # pylint: disable=W0613 """ this function tries to guess which target got used by looking at the different image directories """ target = PortageInstance.getDefaultTarget( category, package, version ) buildroot = os.path.join( ROOTDIR, "build", category, "%s-%s" % ( package, version ) ) if not os.path.exists( buildroot ): return target for directory in os.listdir( buildroot ): if os.path.isdir( os.path.join( buildroot, directory ) ): if directory.startswith( "image" ) and directory != "image": particles = directory.split( '-' )[ 1: ] # the first part should be image- anyway if len(particles) == 3: _platform, _buildType, _target = particles elif len(particles) == 4 and emergePlatform.isCrossCompilingEnabled(): _platform, _buildType, _buildArch, _target = particles elif len(particles) >= 4 and not emergePlatform.isCrossCompilingEnabled(): _platform, _buildType = particles[0:2] _target = '-'.join(particles[2:]) else: return target if _platform == os.getenv( "KDECOMPILER" ) and \ _buildType == os.getenv( "EMERGE_BUILDTYPE" ): return _target return target def getPackageInstance(category, package, buildtarget=None): """return instance of class Package from package file""" return PortageInstance.getPackageInstance(category, package, buildtarget) def getDependencies( category, package, version, runtimeOnly=False ): """returns the dependencies of this package as list of strings: category/package""" if not os.path.isfile( getFilename( category, package, version ) ): utils.die( "package name %s/%s-%s unknown" % ( category, package, version ) ) package, subpackage = getSubPackage( category, package ) if subpackage: utils.debug( "solving package %s/%s/%s-%s %s" % ( category, subpackage, package, version, getFilename( category, package, version ) ), 2 ) else: utils.debug( "solving package %s/%s-%s %s" % ( category, package, version, getFilename( category, package, version ) ), 2 ) subpackage = package deps = [] for pkg in subpackage: mod = __import__( getFilename( category, subpackage, version ) ) if hasattr( mod, 'subinfo' ): info = mod.subinfo() depDict = info.hardDependencies depDict.update( info.dependencies ) depDict.update( info.runtimeDependencies ) if not runtimeOnly: depDict.update( info.buildDependencies ) for line in list(depDict.keys()): (category, package) = line.split( "/" ) version = PortageInstance.getNewestVersion( category, package ) deps.append( [ category, package, version, depDict[ line ] ] ) return deps def solveDependencies( category, package, version, depList, dep_type='both' ): depList.reverse() if ( category == "" ): category = PortageInstance.getCategory( package ) utils.debug( "found package in category %s" % category, 2 ) if ( version == "" ): version = PortageInstance.getNewestVersion( category, package ) utils.debug( "found package with newest version %s" % version, 2 ) pac = DependencyPackage( category, package, version ) depList = pac.getDependencies( depList, dep_type=dep_type ) depList.reverse() return depList def printTargets( category, package, version ): targetsDict = PortageInstance.getAllTargets( category, package, version ) defaultTarget = PortageInstance.getDefaultTarget( category, package, version ) if 'svnHEAD' in targetsDict and not targetsDict['svnHEAD']: del targetsDict['svnHEAD'] targetsDictKeys = list(targetsDict.keys()) targetsDictKeys.sort() for i in targetsDictKeys: if defaultTarget == i: print('*', end=' ') else: print(' ', end=' ') print(i) def readChildren( category, package, version ): identFileName = getFilename( category, package, version ) if not os.path.isfile( identFileName ): utils.die( "package name %s/%s-%s unknown" % ( category, package, version ) ) # if we are an emerge internal package and already in the dictionary, ignore childrens # To avoid possible endless recursion this may be also make sense for all packages if category == internalCategory and identFileName in list(modDict.keys()): return dict(), dict() package, subpackage = getSubPackage( category, package ) if subpackage: utils.debug( "solving package %s/%s/%s-%s %s" % ( category, subpackage, package, version, getFilename( category, package, version ) ), 2 ) else: utils.debug( "solving package %s/%s-%s %s" % ( category, package, version, getFilename( category, package, version ) ), 2 ) if not identFileName in list(modDict.keys()): mod = __import__( identFileName ) modDict[ identFileName ] = mod else: mod = modDict[ identFileName ] if utils.envAsBool('EMERGE_ENABLE_IMPLICID_BUILDTIME_DEPENDENCIES') and hasattr( mod, 'Package' ): _package = mod.Package() subinfo = _package.subinfo elif hasattr( mod, 'subinfo' ): subinfo = mod.subinfo() else: return dict(), dict() runtimeDependencies = subinfo.runtimeDependencies or dict() buildDependencies = subinfo.buildDependencies or dict() # hardDependencies commonDependencies = subinfo.hardDependencies commonDependencies.update( subinfo.dependencies ) runtimeDependencies.update(commonDependencies) buildDependencies.update(commonDependencies) return runtimeDependencies, buildDependencies def isHostBuildEnabled( category, package, version ): """ returns whether this package's host build is enabled. This will only work if isCrossCompilingEnabled() == True """ if emergePlatform.isCrossCompilingEnabled(): mod = __import__( getFilename( category, package, version ) ) if hasattr( mod, 'subinfo' ): info = mod.subinfo() return not info.disableHostBuild else: return False else: utils.die( "This function must not be used outside of cross-compiling environments!" ) def isTargetBuildEnabled( category, package, version ): """ returns whether this package's target build is enabled. This will only work if isCrossCompilingEnabled() == True """ if emergePlatform.isCrossCompilingEnabled(): mod = __import__( getFilename( category, package, version ) ) if hasattr( mod, 'subinfo' ): info = mod.subinfo() return not info.disableTargetBuild else: return False else: utils.die( "This function must not be used outside of cross-compiling environments!" ) def isPackageUpdateable( category, package, version ): utils.debug( "importing file %s" % getFilename( category, package, version ), 2 ) mod = __import__( getFilename( category, package, version ) ) if hasattr( mod, 'subinfo' ): info = mod.subinfo() if len( info.svnTargets ) == 1 and not info.svnTargets[ list(info.svnTargets.keys())[0] ]: return False return len( info.svnTargets ) > 0 else: return False def alwaysTrue( *dummyArgs): """we sometimes need a function that always returns True""" return True def getHostAndTarget( hostEnabled, targetEnabled ): """used for messages""" msg = "" if hostEnabled or targetEnabled: msg += "(" if hostEnabled: msg += "H" if hostEnabled and targetEnabled: msg += "/" if targetEnabled: msg += "T" msg += ")" return msg def printCategoriesPackagesAndVersions( lines, condition, hostEnabled=alwaysTrue, targetEnabled=alwaysTrue ): """prints a number of 'lines', each consisting of category, package and version field""" def printLine( cat, pack, ver, hnt="" ): catlen = 25 packlen = 25 print(cat + " " * ( catlen - len( cat ) ) + pack + " " * ( packlen - len( pack ) ) + ver, hnt) printLine( 'Category', 'Package', 'Version' ) printLine( '--------', '-------', '-------' ) for category, package, version in lines: if emergePlatform.isCrossCompilingEnabled(): msg = getHostAndTarget( hostEnabled( category, package, version ), targetEnabled( category, package, version ) ) else: msg = "" if condition( category, package, version ): printLine( category, package, version, msg ) def printInstallables(): """get all the packages that can be installed""" printCategoriesPackagesAndVersions( PortageInstance.getInstallables(), alwaysTrue ) def printInstalled(): """get all the packages that are already installed""" printCategoriesPackagesAndVersions( PortageInstance.getInstallables(), isInstalled ) def isInstalled( category, package, version, buildtype='' ): """ deprecated, use InstallDB.installdb.isInstalled() instead """ utils.debug( "isInstalled(%s, %s, %s, %s)" % (category, package, version, buildtype), 2 ) # find in old style database path = utils.etcDir() fileName = os.path.join(path,'installed') found = False if os.path.isfile( fileName ): with open( fileName, "rt" ) as f: for line in f.read().splitlines(): if not line: continue # Ignore empty lines (_category, _packageVersion) = line.split( "/" ) (_package, _version) = utils.packageSplit(_packageVersion) if category != '' and version != '' and category == _category and package == _package and version == _version: found = True break elif category == '' and version == '' and package == _package: found = True break utils.debug("...%s found in main database" % (' ' if found else ' not'), 2 ) if found: return True # find in release mode database if buildtype != '': fileName = os.path.join(path,'installed-' + buildtype ) if os.path.isfile( fileName ): with open( fileName, "rt" ) as f: for line in f.read().splitlines(): if not line or line == "": continue (_category, _packageVersion) = line.split( "/" ) (_package, _version) = utils.packageSplit(_packageVersion) if category != '' and version != '' and category == _category and package == _package and version == _version: found = True break elif category == '' and version == '' and package == _package: found = True break utils.debug( "...%s found in %s database" % ( (' ' if found else ' not'), buildtype), 2 ) if found: return True # try to detect packages from the installer binary = utils.checkManifestFile( os.path.join( os.getenv( "KDEROOT" ), "manifest", package + "-" + version + "-bin.ver"), category, package, version ) lib = utils.checkManifestFile( os.path.join( os.getenv( "KDEROOT" ), "manifest", package + "-" + version + "-lib.ver"), category, package, version ) found = found or binary or lib if not utils.envAsBool("EMERGE_VERSIONING", default=True): # check for any installation except data packages if not os.path.exists(os.path.join( os.getenv( "KDEROOT" ), "manifest" ) ): return False if package.endswith( "-src" ): package = package[:-4] for filename in os.listdir( os.path.join( os.getenv( "KDEROOT" ), "manifest" ) ): if filename.startswith( package ) and not \ filename.startswith( package + "-data" ): return True return False def findInstalled( category, package): """ deprecated, use InstallDB.installdb.findInstalled() instead """ fileName = os.path.join(utils.etcDir(), "installed" ) if ( not os.path.isfile( fileName ) ): return None ret = None regexStr = "^%s/%s-(.*)$" % ( category, re.escape(package) ) regex = re.compile( regexStr ) with open( fileName, "rt" ) as f: for line in f.read().splitlines(): match = regex.match( line ) if match: utils.debug( "found: " + match.group(1), 2 ) ret = match.group(1) return ret def addInstalled( category, package, version, buildtype='' ): """ deprecated, use InstallDB.installdb.addInstalled() instead """ utils.debug( "addInstalled called", 2 ) # write a line to etc/portage/installed, # that contains category/package-version path = os.path.join(utils.etcDir() ) if ( not os.path.isdir( path ) ): os.makedirs( path ) if buildtype != '': fileName = 'installed-' + buildtype else: fileName = 'installed' utils.debug("installing package %s - %s into %s" % (package, version, fileName), 2) if( os.path.isfile( os.path.join( path, fileName ) ) ): with open( os.path.join( path, fileName ), "rt" ) as f: for line in f: if line.startswith( "%s/%s-%s" % ( category, package, version) ): utils.warning( "version already installed" ) return elif line.startswith( "%s/%s-" % ( category, package ) ): utils.die( "Already installed, this should not happen" ) with open( os.path.join( path, fileName ), "at" ) as f: f.write( "%s/%s-%s\r\n" % ( category, package, version ) ) def remInstalled( category, package, version, buildtype='' ): """ deprecated, use InstallDB.installdb.remInstalled() instead """ utils.debug( "remInstalled called", 2 ) if buildtype != '': fileName = 'installed-' + buildtype else: fileName = 'installed' utils.debug("removing package %s - %s from %s" % (package, version, fileName), 2) dbFileName = os.path.join(utils.etcDir(), fileName ) tmpdbfile = os.path.join(utils.etcDir(), "TMPinstalled" ) found = False if os.path.exists( dbFileName ): with open( dbFileName, "rt" ) as dbFile: with open( tmpdbfile, "wt" ) as tfile: for line in dbFile: ## \todo the category should not be part of the search string ## because otherwise it is not possible to unmerge package using ## the same name but living in different categories if not line.startswith("%s/%s" % ( category, package ) ): tfile.write( line ) else: found = True os.remove( dbFileName ) os.rename( tmpdbfile, dbFileName ) return found def getPackagesCategories(packageName, defaultCategory = None): utils.debug( "getPackagesCategories for package name %s" % packageName, 1 ) if defaultCategory is None: if "EMERGE_DEFAULTCATEGORY" in os.environ: defaultCategory = os.environ["EMERGE_DEFAULTCATEGORY"] else: defaultCategory = "kde" packageList, categoryList = [], [] if len( packageName.split( "/" ) ) == 1: if PortageInstance.isCategory( packageName ): utils.debug( "isCategory=True", 2 ) packageList = PortageInstance.getAllPackages( packageName ) categoryList = [ packageName ] * len(packageList) else: utils.debug( "isCategory=False", 2 ) if PortageInstance.isCategory( defaultCategory ) and PortageInstance.isPackage( defaultCategory, packageName ): # prefer the default category packageList = [ packageName ] categoryList = [ defaultCategory ] else: if PortageInstance.getCategory( packageName ): packageList = [ packageName ] categoryList = [ PortageInstance.getCategory( packageName ) ] else: utils.warning( "unknown category or package: %s" % packageName ) elif len( packageName.split( "/" ) ) == 2: [ cat, pac ] = packageName.split( "/" ) if PortageInstance.isCategory( cat ): categoryList = [ cat ] else: utils.warning( "unknown category %s; ignoring package %s" % ( cat, packageName ) ) if len( categoryList ) > 0 and PortageInstance.isPackage( categoryList[0], pac ): packageList = [ pac ] if len( categoryList ) and len( packageList ): utils.debug( "added package %s/%s" % ( categoryList[0], pac ), 2 ) else: utils.debug( "ignoring package %s" % packageName ) else: utils.error( "unknown packageName" ) return packageList, categoryList