diff --git a/bin/Blueprints/CraftDependencyPackage.py b/bin/Blueprints/CraftDependencyPackage.py index cf328d1ac..0f9530ebe 100644 --- a/bin/Blueprints/CraftDependencyPackage.py +++ b/bin/Blueprints/CraftDependencyPackage.py @@ -1,108 +1,111 @@ from collections import OrderedDict from enum import unique, Enum, IntFlag from Blueprints.CraftPackageObject import CraftPackageObject, BlueprintException from Blueprints.CraftVersion import CraftVersion from CraftCore import CraftCore @unique class DependencyType(IntFlag): Runtime = 0x1 << 0 Buildtime = 0x1 << 1 # TODO: rename as we now have more build types Both = Runtime | Buildtime Packaging = 0x1 << 3 - All = ~0 + Categories = 0x1 << 4 + + # TODO: maybe All is not the best name, anyway including categories in a normal scenario makes no sense + All = ~0 & ~Categories class CraftDependencyPackage(CraftPackageObject): _packageCache = dict() @unique class State(Enum): Unvisited = 0 Visiting = 1 Visited = 2 def __init__(self, path): CraftPackageObject.__init__(self, path) self._depenendencyType = None self.dependencies = [] # tuple (name, required version) self.state = CraftDependencyPackage.State.Unvisited @property def depenendencyType(self): return self._depenendencyType @depenendencyType.setter def depenendencyType(self, depenendencyType): if self._depenendencyType != depenendencyType: self._depenendencyType = depenendencyType self.dependencies = [] self.__resolveDependencies() def __resolveDependencies(self): CraftCore.log.debug(f"resolving package {self}") if not self.isCategory(): subinfo = self.subinfo if self.depenendencyType & DependencyType.Runtime: self.dependencies.extend(self.__readDependenciesForChildren(subinfo.runtimeDependencies.items())) if self.depenendencyType & DependencyType.Buildtime: self.dependencies.extend(self.__readDependenciesForChildren(subinfo.buildDependencies.items())) if self.depenendencyType & DependencyType.Packaging: self.dependencies.extend(self.__readDependenciesForChildren(subinfo.packagingDependencies.items())) else: self.dependencies.extend(self.__readDependenciesForChildren([(x, None) for x in self.children.values()])) - def __readDependenciesForChildren(self, deps): + def __readDependenciesForChildren(self, deps : [(str, str)]) -> []: children = [] if deps: for packaheName, requiredVersion in deps: if (packaheName, self.depenendencyType) not in CraftDependencyPackage._packageCache: package = CraftPackageObject.get(packaheName) if not package: raise BlueprintException(f"Failed to resolve {packaheName} as a dependency of {self}", self) if requiredVersion and requiredVersion != "default" and CraftVersion(package.version) < CraftVersion(requiredVersion): raise BlueprintException(f"{self} requries {package} version {requiredVersion!r} but {package.version!r} is installed", self) p = CraftDependencyPackage(package) CraftCore.log.debug(f"adding package {packaheName}") CraftDependencyPackage._packageCache[(packaheName, self.depenendencyType)] = p p.depenendencyType = self.depenendencyType else: p = CraftDependencyPackage._packageCache[(packaheName, self.depenendencyType)] children.append(p) return children def __getDependencies(self, depenendencyType, ignoredPackages): """ returns all dependencies """ if self.isIgnored(): return [] self.depenendencyType = depenendencyType depList = [] self.state = CraftDependencyPackage.State.Visiting for p in self.dependencies: if p.state != CraftDependencyPackage.State.Unvisited: continue if not p.isIgnored() \ and (not ignoredPackages or p.path not in ignoredPackages): depList.extend(p.__getDependencies(depenendencyType & ~DependencyType.Packaging, ignoredPackages)) if depenendencyType & DependencyType.Packaging: depList.extend(p.__getDependencies(depenendencyType, ignoredPackages)) if self.state != CraftDependencyPackage.State.Visited: self.state = CraftDependencyPackage.State.Visited - if not self.isCategory(): + if not self.isCategory() or depenendencyType & DependencyType.Categories: depList.append(self) return list(OrderedDict.fromkeys(depList)) def getDependencies(self, depType=DependencyType.All, ignoredPackages=None): self.depenendencyType = depType for p in CraftDependencyPackage._packageCache.values(): #reset visited state p.state = CraftDependencyPackage.State.Unvisited return self.__getDependencies(depType, ignoredPackages) diff --git a/bin/Blueprints/CraftPackageObject.py b/bin/Blueprints/CraftPackageObject.py index 7ec31ff82..afc9f5c4f 100644 --- a/bin/Blueprints/CraftPackageObject.py +++ b/bin/Blueprints/CraftPackageObject.py @@ -1,358 +1,361 @@ #!/usr/bin/env python import copy import configparser import importlib import os import re import utils from CraftCore import CraftCore from CraftStandardDirs import CraftStandardDirs from CraftOS.osutils import OsUtils class CategoryPackageObject(object): def __init__(self, localPath : str): self.localPath = localPath - self.desctiption = "" + self.description = "" + self.webpage = "" self.platforms = CraftCore.compiler.Platforms.All self.compiler = CraftCore.compiler.Compiler.All self.pathOverride = None self.valid = False ini = os.path.join(self.localPath, "info.ini") if os.path.exists(ini): self.valid = True info = configparser.ConfigParser() info.read(ini) - self.desctiption = info["General"].get("description", "") - platform = set(CraftCore.settings._parseList(info["General"].get("platforms", ""))) + general = info["General"] + self.description = general.get("description", "") + self.webpage = general.get("webpage", "") + platform = set(CraftCore.settings._parseList(general.get("platforms", ""))) if platform: self.platforms = CraftCore.compiler.Platforms.NoPlatform for p in platform: self.platforms |= CraftCore.compiler.Platforms.fromString(p) - compiler = set(CraftCore.settings._parseList(info["General"].get("compiler", ""))) + compiler = set(CraftCore.settings._parseList(general.get("compiler", ""))) if compiler: self.compiler = CraftCore.compiler.Compiler.NoCompiler for c in compiler: self.compiler |= CraftCore.compiler.Compiler.fromString(c) - self.pathOverride = info["General"].get("pathOverride", None) + self.pathOverride = general.get("pathOverride", None) @property def isActive(self) -> bool: if not CraftCore.compiler.platform & self.platforms: CraftCore.log.debug(f"{self.localPath}, is not supported on {CraftCore.compiler.platform!r}, supported platforms {self.platforms!r}") return False if not CraftCore.compiler.compiler & self.compiler: CraftCore.log.debug(f"{self.localPath}, is not supported on {CraftCore.compiler._compiler!r}, supported compiler {self.compiler!r}") return False return True class CraftPackageObject(object): __rootPackage = None __rootDirectories = [] # list of all leaves, unaffected by overrides etc _allLeaves = {} _recipes = {}#all recipes, for lookup by package name IgnoredDirectories = {"__pycache__"} Ignores = re.compile("a^") @staticmethod def _isDirIgnored(d): return d.startswith(".") or d in CraftPackageObject.IgnoredDirectories def __init__(self, other=None, parent=None): if isinstance(other, CraftPackageObject): self.__dict__ = other.__dict__ return if other and not parent: raise Exception("Calling CraftPackageObject(str) directly is not supported," " use CraftPackageObject.get(str) instead.") self.parent = parent self.name = other self.children = {} self.source = None self.categoryInfo = None self._version = None self._instance = None self.__path = None @property def parent(self): return self._parent @parent.setter def parent(self, parent): self._parent = parent self.__path = None @property def path(self): if not self.__path: if not self.name: return None if self.name == "/": self.__path = self.name elif self.parent.path == "/": self.__path = self.name else: self.__path = "/".join([self.parent.path, self.name]) return self.__path @staticmethod def get(path): if isinstance(path, CraftPackageObject): return path root = CraftPackageObject.root() package = None if path in CraftPackageObject._recipes: packages = CraftPackageObject._recipes[path] if len(packages) > 1: CraftCore.log.info(f"Found multiple recipes for {path}") for p in packages: CraftCore.log.info(f"{p}: {p.source}") CraftCore.log.info(f"Please use the full path to the recipe.") exit(1) package = packages[0] else: if path == "/": return root else: components = path.split("/") package = root for part in components: package = package.children.get(part, None) if not package: return None return package @staticmethod def _expandChildren(path, parent, blueprintRoot): if path: path = utils.normalisePath(path) name = path.rsplit("/", 1)[-1] if name in parent.children: package = parent.children[name] else: package = CraftPackageObject(name, parent) elif blueprintRoot: path = blueprintRoot package = parent else: raise Exception("Unreachable") if not package.categoryInfo: package.categoryInfo = CategoryPackageObject(path) if not package.categoryInfo.valid and package.parent: package.categoryInfo = copy.copy(package.parent.categoryInfo) for f in os.listdir(path): fPath = os.path.abspath(os.path.join(path, f)) if os.path.isdir(fPath): if not CraftPackageObject._isDirIgnored(f): child = CraftPackageObject._expandChildren(fPath, package, blueprintRoot) if child: if f in package.children: existingNode = package.children[f] if not existingNode.isCategory(): CraftCore.log.warning( f"Blueprint clash detected: Ignoring {child.source} in favour of {existingNode.source}") continue else: #merge with existing node existingNode.children.update(child.children) else: package.children[f] = child elif f.endswith(".py"): if package.source: raise BlueprintException(f"Multiple py files in one directory: {package.source} and {f}", package) if f[:-3] != package.name: raise BlueprintException(f"Recipes must match the name of the directory: {fPath}", package) package.source = fPath CraftPackageObject._allLeaves[package.path] = package if package.children and package.source: raise BlueprintException(f"{package} has has children but also a recipe {package.source}!", package) if path != blueprintRoot: if not package.source and not package.children: if os.listdir(path) in [["__pycache__"], []]: # the recipe was removed utils.rmtree(path) else: CraftCore.log.warning(f"Found an dead branch in {blueprintRoot}/{package.path}\n" f"You might wan't to run \"git clean -xdf\" in that directry.") return None return package @staticmethod def rootDirectories(): # this function should return all currently set blueprint directories if not CraftPackageObject.__rootDirectories: rootDirs = {utils.normalisePath(CraftStandardDirs.craftRepositoryDir())} if ("Blueprints", "Locations") in CraftCore.settings: for path in CraftCore.settings.getList("Blueprints", "Locations"): rootDirs.add(utils.normalisePath(path)) if os.path.isdir(CraftStandardDirs.blueprintRoot()): for f in os.listdir(CraftStandardDirs.blueprintRoot()): if CraftPackageObject._isDirIgnored(f): continue rootDirs.add(utils.normalisePath(os.path.join(CraftStandardDirs.blueprintRoot(), f))) CraftCore.log.debug(f"Craft BlueprintLocations: {rootDirs}") CraftPackageObject.__rootDirectories = list(rootDirs) return CraftPackageObject.__rootDirectories @staticmethod def bootstrapping() -> bool: return len(CraftPackageObject.rootDirectories()) == 1 @staticmethod def __regiserNodes(package): # danger, we detach here for child in list(package.children.values()): # hash leaves for direct acces if not child.isCategory(): if not (not child.categoryInfo.isActive and child.categoryInfo.pathOverride): CraftCore.log.debug(f"Adding package {child.source}") if child.name not in CraftPackageObject._recipes: CraftPackageObject._recipes[child.name] = [] CraftPackageObject._recipes[child.name].append(child) else: if child.categoryInfo.pathOverride: # override path existingNode = CraftPackageObject.get(child.categoryInfo.pathOverride) if not existingNode: raise BlueprintNotFoundException(child.categoryInfo.pathOverride) for nodeName, node in child.children.items(): # reparent the packages if nodeName in existingNode.children: if not node.categoryInfo.isActive: # don't reparent as we would override the actual package continue else: CraftCore.log.debug(f"Overriding {existingNode.children[nodeName].path} with {node.path} ") node.parent = existingNode child.parent.children[nodeName] = node CraftPackageObject.__regiserNodes(child) @staticmethod def root(): if not CraftPackageObject.__rootPackage: if ("Blueprints", "Ignores") in CraftCore.settings: CraftPackageObject.Ignores = re.compile("|".join([f"^{entry}$" for entry in CraftCore.settings.get("Blueprints", "Ignores").split(";")])) CraftPackageObject.__rootPackage = root = CraftPackageObject() root.name = "/" for blueprintRoot in CraftPackageObject.rootDirectories(): if not os.path.isdir(blueprintRoot): CraftCore.log.warning(f"{blueprintRoot} does not exist") continue blueprintRoot = utils.normalisePath(os.path.abspath(blueprintRoot)) # create a dummy package to load its children child = CraftPackageObject._expandChildren(None, root, blueprintRoot) root.children.update(child.children) CraftPackageObject.__regiserNodes(root) return CraftPackageObject.__rootPackage @property def instance(self): if not self._instance: CraftCore.log.debug(f"module to import: {self.source} {self.path}") modulename = os.path.splitext(os.path.basename(self.source))[0].replace('.', '_') loader = importlib.machinery.SourceFileLoader(modulename, self.source) try: mod = loader.load_module() except Exception as e: raise BlueprintException(f"Failed to load file {self.source}", self, e) if not mod is None: mod.CRAFT_CURRENT_MODULE = self pack = mod.Package() self._instance = pack else: raise BlueprintException("Failed to find package", self) return self._instance @property def isInstalled(self) -> bool: return len(CraftCore.installdb.getInstalledPackages(self)) == 1 @property def subinfo(self): return self.instance.subinfo def isCategory(self): return not self.source def isIgnored(self): if self.categoryInfo and not self.categoryInfo.isActive: return True import options if not self.path: return False ignored = options.UserOptions.get(self).ignored if ignored is not None: return ignored ignored = self.path and CraftPackageObject.Ignores.match(self.path) if ignored: CraftCore.log.warning(f"You are using the deprecated Ignore setting:\n" f"[Blueprints]\n" f"Ignores={self.path}\n\n" f"Please use BlueprintSettings.ini\n" f"[{self.path}]\n" f"ignored = True") return ignored @property def version(self): if self.isCategory(): return None if not self._version: self._version = self.instance.version return self._version def __eq__(self, other): if isinstance(other, CraftPackageObject): return self.path == other.path return self.path == other def __str__(self): return self.path or "None" def __hash__(self): return self.path.__hash__() @staticmethod def installables(): #ensure that everything is loaded CraftPackageObject.root() recipes = [] for p in CraftPackageObject._recipes.values(): recipes.extend(p) return recipes class BlueprintException(Exception): def __init__(self, message, package, exception=None): Exception.__init__(self, message) self.package = package self.exception = exception def __str__(self): return f"{self.package.source or self.package} failed:\n{Exception.__str__(self)}" class BlueprintNotFoundException(Exception): def __init__(self, packageName, message=None): Exception.__init__(self) self.packageName = packageName self.message = message def __str__(self): if self.message: return self.message else: return f"Failed to find {self.packageName}: {Exception.__str__(self)}" diff --git a/bin/blueprintSearch.py b/bin/blueprintSearch.py index dc4f7f3e4..28d8438c1 100644 --- a/bin/blueprintSearch.py +++ b/bin/blueprintSearch.py @@ -1,107 +1,121 @@ import InstallDB import utils from Blueprints.CraftPackageObject import * +from Blueprints.CraftDependencyPackage import * from Blueprints.CraftVersion import CraftVersion from Utils import CraftTimer class SeachPackage(object): def __init__(self, package): self.path = package.path self.name = package.name - self.webpage = package.subinfo.webpage - self.description = package.subinfo.description - self.tags = package.subinfo.tags + self.displayName = self.name + self.webpage = None + self.description = "" + self.tags = "" + self.availableVersions = None + if not package.isCategory(): + self.displayName = package.subinfo.displayName + self.webpage = package.subinfo.webpage + self.description = package.subinfo.description + self.tags = package.subinfo.tags + + versions = list(package.subinfo.svnTargets.keys()) + list(package.subinfo.targets.keys()) + versions.sort(key=lambda x: CraftVersion(x)) + self.availableVersions = ", ".join(versions) + else: + self.description = package.categoryInfo.description + self.webpage = package.categoryInfo.webpage - versions = list(package.subinfo.svnTargets.keys()) + list(package.subinfo.targets.keys()) - versions.sort(key=lambda x: CraftVersion(x)) - self.availableVersions = ", ".join(versions) @property def package(self): # we can't cache the whole instance out = CraftPackageObject.get(self.path) if not out: CraftCore.log.error("Cache corrupted") CraftCore.cache.clear() exit(1) return out def __str__(self): installed = CraftCore.installdb.getInstalledPackages(self.package) version = None revision = None if installed: if (len(installed) > 1): raise Exception("Multiple installs are not supported") version = installed[0].getVersion() or None revision = installed[0].getRevision() or None latestVersion = self.package.version return f"""\ {self.package} - Name: {self.package.subinfo.displayName} + Name: {self.displayName} BlueprintPath: {self.package.source} Homepage: {self.webpage} Description: {self.description} Tags: {self.tags} Latest version: {latestVersion} Installed versions: {version} Installed revision: {revision} Available versions: {self.availableVersions} """ def packages(): if not CraftCore.cache.availablePackages: CraftCore.cache.availablePackages = [] CraftCore.log.info("Updating search cache:") - total = len(CraftPackageObject.installables()) - for p in CraftPackageObject.installables(): + root = CraftDependencyPackage(CraftPackageObject.root()) + packages = root.getDependencies(DependencyType.All | DependencyType.Categories) + total = len(packages) + for p in packages: package = SeachPackage(p) CraftCore.cache.availablePackages.append(package) percent = int(len(CraftCore.cache.availablePackages) / total * 100) utils.printProgress(percent) utils.printProgress(100) CraftCore.log.info("") return CraftCore.cache.availablePackages def printSearch(search_package, maxDist=2): searchPackageLower = search_package.lower() isPath = "/" in searchPackageLower with CraftTimer.Timer("Search", 0) as timer: similar = [] match = None package_re = re.compile(f".*{search_package}.*", re.IGNORECASE) for searchPackage in packages(): packageString = searchPackage.path if isPath else searchPackage.name levDist = abs(len(searchPackageLower) - len(packageString)) if levDist <= maxDist: levDist = utils.levenshtein(searchPackageLower, packageString.lower()) if levDist == 0: match = (levDist, searchPackage) break elif package_re.match(searchPackage.path): similar.append((levDist - maxDist, searchPackage)) elif len(packageString) > maxDist and levDist <= maxDist: similar.append((levDist, searchPackage)) else: if package_re.match(searchPackage.description) or \ package_re.match(searchPackage.tags): similar.append((100, searchPackage)) if match == None: if len(similar) > 0: CraftCore.log.info(f"Craft was unable to find {search_package}, similar packages are:") similar.sort(key=lambda x: x[0]) else: CraftCore.log.info(f"Craft was unable to find {search_package}") else: CraftCore.log.info(f"Package {search_package} found:") similar = [match] for levDist, searchPackage in similar: CraftCore.log.debug((vars(searchPackage), levDist)) CraftCore.log.info(searchPackage)