diff --git a/bin/Blueprints/CraftPackageObject.py b/bin/Blueprints/CraftPackageObject.py index 6dbb465c7..52c6c7590 100644 --- a/bin/Blueprints/CraftPackageObject.py +++ b/bin/Blueprints/CraftPackageObject.py @@ -1,393 +1,389 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import copy import configparser import importlib import os import re import utils from CraftCore import CraftCore from CraftStandardDirs import CraftStandardDirs from CraftOS.osutils import OsUtils class CategoryPackageObject(object): def __init__(self, localPath : str): self.localPath = localPath self.description = "" self.webpage = "" self.displayName = "" self.tags = "" self.platforms = CraftCore.compiler.Platforms.All self.compiler = CraftCore.compiler.Compiler.All self.pathOverride = None self.valid = False ini = os.path.join(self.localPath, "info.ini") if os.path.exists(ini): self.valid = True info = configparser.ConfigParser() info.read(ini) general = info["General"] self.displayName = general.get("displayName", "") self.description = general.get("description", "") self.tags = general.get("tags", "") self.webpage = general.get("webpage", "") platform = set(CraftCore.settings._parseList(general.get("platforms", ""))) if platform: self.platforms = CraftCore.compiler.Platforms.NoPlatform for p in platform: self.platforms |= CraftCore.compiler.Platforms.fromString(p) compiler = set(CraftCore.settings._parseList(general.get("compiler", ""))) if compiler: self.compiler = CraftCore.compiler.Compiler.NoCompiler for c in compiler: self.compiler |= CraftCore.compiler.Compiler.fromString(c) self.pathOverride = general.get("pathOverride", None) @property def isActive(self) -> bool: if not CraftCore.compiler.platform & self.platforms: CraftCore.log.debug(f"{self.localPath}, is not supported on {CraftCore.compiler.platform!r}, supported platforms {self.platforms!r}") return False if not CraftCore.compiler.compiler & self.compiler: CraftCore.log.debug(f"{self.localPath}, is not supported on {CraftCore.compiler._compiler!r}, supported compiler {self.compiler!r}") return False return True class CraftPackageObject(object): __rootPackage = None __rootDirectories = [] # list of all leaves, unaffected by overrides etc _allLeaves = {} _recipes = {}#all recipes, for lookup by package name IgnoredDirectories = {"__pycache__"} Ignores = re.compile("a^") @staticmethod def _isDirIgnored(d): return d.startswith(".") or d in CraftPackageObject.IgnoredDirectories def __init__(self, other=None, parent=None): if isinstance(other, CraftPackageObject): self.__dict__ = other.__dict__ return if other and not parent: raise Exception("Calling CraftPackageObject(str) directly is not supported," " use CraftPackageObject.get(str) instead.") self.parent = parent self.name = other self.filePath = "" self.children = {} self.source = None self.categoryInfo = None self._version = None self._instance = None self.__path = None @property def parent(self): return self._parent @parent.setter def parent(self, parent): self._parent = parent self.__path = None @property def path(self): if not self.__path: if not self.name: return None if self.name == "/": self.__path = self.name elif self.parent.path == "/": self.__path = self.name else: self.__path = "/".join([self.parent.path, self.name]) return self.__path @staticmethod def get(path): if isinstance(path, CraftPackageObject): return path root = CraftPackageObject.root() package = None if path in CraftPackageObject._recipes: packages = CraftPackageObject._recipes[path] if len(packages) > 1: CraftCore.log.info(f"Found multiple recipes for {path}") for p in packages: CraftCore.log.info(f"{p}: {p.source}") CraftCore.log.info(f"Please use the full path to the recipe.") exit(1) package = packages[0] else: if path == "/": return root else: components = path.split("/") package = root for part in components: package = package.children.get(part, None) if not package: return None return package @staticmethod def _expandChildren(path, parent, blueprintRoot): if path: path = utils.normalisePath(path) name = path.rsplit("/", 1)[-1] if name in parent.children: package = parent.children[name] else: package = CraftPackageObject(name, parent) package.filePath = path elif blueprintRoot: path = blueprintRoot package = parent else: raise Exception("Unreachable") if not package.categoryInfo: package.categoryInfo = CategoryPackageObject(path) if not package.categoryInfo.valid and package.parent: # we actually need a copy package.categoryInfo = copy.copy(package.parent.categoryInfo) if not package.categoryInfo.valid: package.categoryInfo = CategoryPackageObject(blueprintRoot) for f in os.listdir(path): fPath = os.path.abspath(os.path.join(path, f)) if os.path.isdir(fPath): if not CraftPackageObject._isDirIgnored(f): child = CraftPackageObject._expandChildren(fPath, package, blueprintRoot) if child: if f in package.children: existingNode = package.children[f] if not existingNode.isCategory(): CraftCore.log.warning( f"Blueprint clash detected: Ignoring {child.source} in favour of {existingNode.source}") continue else: #merge with existing node existingNode.children.update(child.children) else: package.children[f] = child elif f.endswith(".py"): if package.source: raise BlueprintException(f"Multiple py files in one directory: {package.source} and {f}", package) if f[:-3] != package.name: raise BlueprintException(f"Recipes must match the name of the directory: {fPath}", package) package.source = fPath CraftPackageObject._allLeaves[package.path] = package if package.children and package.source: raise BlueprintException(f"{package} has has children but also a recipe {package.source}!", package) if path != blueprintRoot: if not package.source and not package.children: if os.listdir(path) in [["__pycache__"], []]: # the recipe was removed utils.rmtree(path) else: CraftCore.log.warning(f"Found an dead branch in {blueprintRoot}/{package.path}\n" f"You might wan't to run \"git clean -xdf\" in that directry.") return None return package @staticmethod def rootDirectories(): # this function should return all currently set blueprint directories if not CraftPackageObject.__rootDirectories: rootDirs = {utils.normalisePath(CraftStandardDirs.craftRepositoryDir())} if ("Blueprints", "Locations") in CraftCore.settings: for path in CraftCore.settings.getList("Blueprints", "Locations"): rootDirs.add(utils.normalisePath(path)) if os.path.isdir(CraftStandardDirs.blueprintRoot()): for f in os.listdir(CraftStandardDirs.blueprintRoot()): if CraftPackageObject._isDirIgnored(f): continue rootDirs.add(utils.normalisePath(os.path.join(CraftStandardDirs.blueprintRoot(), f))) CraftCore.log.debug(f"Craft BlueprintLocations: {rootDirs}") CraftPackageObject.__rootDirectories = list(rootDirs) return CraftPackageObject.__rootDirectories - @staticmethod - def bootstrapping() -> bool: - return len(CraftPackageObject.rootDirectories()) == 1 - @staticmethod def __regiserNodes(package): # danger, we detach here for child in list(package.children.values()): # hash leaves for direct acces if not child.isCategory(): if not (not child.categoryInfo.isActive and child.categoryInfo.pathOverride): CraftCore.log.debug(f"Adding package {child.source}") if child.name not in CraftPackageObject._recipes: CraftPackageObject._recipes[child.name] = [] CraftPackageObject._recipes[child.name].append(child) else: if child.categoryInfo.pathOverride: # override path existingNode = CraftPackageObject.get(child.categoryInfo.pathOverride) if not existingNode: raise BlueprintNotFoundException(child.categoryInfo.pathOverride) for nodeName, node in child.children.items(): # reparent the packages if nodeName in existingNode.children: if not node.categoryInfo.isActive: # don't reparent as we would override the actual package continue else: old = existingNode.children[nodeName] CraftCore.log.debug(f"Overriding {old.path}({old.filePath}) with {node.path}") node.parent = existingNode child.parent.children[nodeName] = node CraftPackageObject.__regiserNodes(child) @staticmethod def root(): if not CraftPackageObject.__rootPackage: if ("Blueprints", "Ignores") in CraftCore.settings: CraftPackageObject.Ignores = re.compile("|".join([f"^{entry}$" for entry in CraftCore.settings.get("Blueprints", "Ignores").split(";")])) CraftPackageObject.__rootPackage = root = CraftPackageObject() root.name = "/" for blueprintRoot in CraftPackageObject.rootDirectories(): if not os.path.isdir(blueprintRoot): CraftCore.log.warning(f"{blueprintRoot} does not exist") continue blueprintRoot = utils.normalisePath(os.path.abspath(blueprintRoot)) # create a dummy package to load its children child = CraftPackageObject._expandChildren(None, root, blueprintRoot) root.children.update(child.children) CraftPackageObject.__regiserNodes(root) return CraftPackageObject.__rootPackage @property def instance(self): if not self._instance: CraftCore.log.debug(f"module to import: {self.source} {self.path}") modulename = os.path.splitext(os.path.basename(self.source))[0].replace('.', '_') loader = importlib.machinery.SourceFileLoader(modulename, self.source) try: mod = loader.load_module() except Exception as e: raise BlueprintException(f"Failed to load file {self.source}", self, e) if not mod is None: mod.CRAFT_CURRENT_MODULE = self pack = mod.Package() self._instance = pack else: raise BlueprintException("Failed to find package", self) return self._instance @property def isInstalled(self) -> bool: return len(CraftCore.installdb.getInstalledPackages(self)) == 1 @property def subinfo(self): return self.instance.subinfo def isCategory(self): return not self.source def isIgnored(self): if self.categoryInfo and not self.categoryInfo.isActive: return True import options if not self.path: return False ignored = options.UserOptions.get(self).ignored if ignored is not None: return ignored ignored = self.path and CraftPackageObject.Ignores.match(self.path) if ignored: CraftCore.log.warning(f"You are using the deprecated Ignore setting:\n" f"[Blueprints]\n" f"Ignores={self.path}\n\n" f"Please use BlueprintSettings.ini\n" f"[{self.path}]\n" f"ignored = True") return ignored @property def version(self): if self.isCategory(): return None if not self._version: self._version = self.instance.version return self._version def __eq__(self, other): if isinstance(other, CraftPackageObject): return self.path == other.path return self.path == other def __str__(self): return self.path or "None" def __hash__(self): return self.path.__hash__() def allChildren(self): recipes = [] for p in self.children.values(): recipes.append(p) recipes.extend(p.allChildren()) return recipes class BlueprintException(Exception): def __init__(self, message, package, exception=None): Exception.__init__(self, message) self.package = package self.exception = exception def __str__(self): return f"{self.package.source or self.package} failed:\n{Exception.__str__(self)}" class BlueprintNotFoundException(Exception): def __init__(self, packageName, message=None): Exception.__init__(self) self.packageName = packageName self.message = message def __str__(self): if self.message: return self.message else: return f"Failed to find {self.packageName}: {Exception.__str__(self)}" diff --git a/bin/options.py b/bin/options.py index 31ff0f9c2..084126079 100644 --- a/bin/options.py +++ b/bin/options.py @@ -1,430 +1,429 @@ ## @package property handling # # (c) copyright 2009-2011 Ralf Habacker # # # import utils from CraftConfig import * from CraftCore import CraftCore from Blueprints.CraftPackageObject import * from CraftDebug import deprecated import configparser import atexit import copy class UserOptions(object): class UserOptionsSingleton(object): _instance = None @property def __header(self): return """\ # The content of this file is partly autogenerated # You can modify values and add settings for your blueprints # Common settings available for all blueprints are: # ignored: [True|False] # version: some version # # use the same url as defined for the target but checks out a different branch # branch: str # patchLevel: int # buildTests: [True|False] # buildStatic: [True|False] # # arguments passed to the configure step # args: str # # Example: ## [libs] ## ignored = True ## ## [lib/qt5] ## version = 5.9.3 ## ignored = False ## withMySQL = True ## ## [kde/pim/akonadi] ## args = -DAKONADI_BUILD_QSQLITE=On ## # # Settings are inherited, so you can set them for a whole sub branch or a single blueprint. # While blueprint from [libs] are all ignored blueprint from [libs/qt5] are not. # """ def __init__(self): self.cachedOptions = {} self.packageOptions = {} self.registeredOptions = {} self.path = CraftCore.settings.get("Blueprints", "Settings", os.path.join(CraftCore.standardDirs.etcDir(), "BlueprintSettings.ini")) self.settings = configparser.ConfigParser(allow_no_value=True) self.settings.optionxform = str if os.path.isfile(self.path): self.settings.read(self.path, encoding="utf-8") def initPackage(self, option): path = option._package.path if not self.settings.has_section(path): self.settings.add_section(path) settings = self.settings[path] return settings def toBool(self, x : str) -> bool: if not x: return False return self.settings._convert_to_boolean(x) @staticmethod @atexit.register def __dump(): instance = UserOptions.UserOptionsSingleton._instance if instance: try: with open(instance.path, "wt", encoding="utf-8") as configfile: print(instance.__header, file=configfile) instance.settings.write(configfile) except Exception as e: CraftCore.log.warning(f"Failed so save {instance.path}: {e}") @staticmethod def instance(): if not UserOptions.UserOptionsSingleton._instance: UserOptions.UserOptionsSingleton._instance = UserOptions.UserOptionsSingleton() return UserOptions.UserOptionsSingleton._instance def __init__(self, package): self._cachedFromParent = {} self._package = package _register = self.registerOption _convert = self._convert _register("version", str, permanent=False) _register("branch", str, permanent=False) _register("patchLevel", int, permanent=False) _register("ignored", bool, permanent=False) _register("buildTests", bool, permanent=False) _register("buildStatic",bool, permanent=False) _register("args", "", permanent=False) settings = UserOptions.instance().settings if settings.has_section(package.path): _registered = UserOptions.instance().registeredOptions[package.path] for k, v in settings[package.path].items(): if k in _registered: v = _convert(_registered[k], v) setattr(self, k, v) @staticmethod def get(package): _instance = UserOptions.instance() packagePath = package.path if packagePath in _instance.cachedOptions: option = _instance.cachedOptions[packagePath] else: option = UserOptions(package) _instance.cachedOptions[packagePath] = option return option def _convert(self, valA, valB): """ Converts valB to type(valA) """ try: if valA is None: return valB _type = valA if callable(valA) else type(valA) if _type == type(valB): return valB if _type is bool: return UserOptions.instance().toBool(valB) return _type(valB) except Exception as e: CraftCore.log.error(f"Can't convert {valB} to {_type.__name__}") raise e @staticmethod def setOptions(optionsIn): packageOptions = UserOptions.instance().packageOptions sectionRe = re.compile(r"\[([^\[\]]+)\](.*)") for o in optionsIn: key, value = o.split("=", 1) key, value = key.strip(), value.strip() match = sectionRe.findall(key) if match: # TODO: move out of options.py section, key = match[0] CraftCore.log.info(f"setOptions: [{section}]{key} = {value}") CraftCore.settings.set(section, key, value) else: package, key = key.split(".", 1) if CraftPackageObject.get(package): if package not in packageOptions: packageOptions[package] = {} CraftCore.log.info(f"setOptions: BlueprintSettings.ini [{package}]{key} = {value}") packageOptions[package][key] = value - elif not CraftPackageObject.bootstrapping(): - # in case we are bootstrapping Craft, we might not know that package yet + else: raise BlueprintNotFoundException(package, f"Package {package} not found, failed to set option {key} = {value}") @staticmethod def addPackageOption(package : CraftPackageObject, key : str, value : str) -> None: if package.path not in UserOptions.instance().packageOptions: UserOptions.instance().packageOptions[package.path] = {} UserOptions.instance().packageOptions[package.path][key] = value def setOption(self, key, value) -> bool: _instance = UserOptions.instance() package = self._package if package.path not in _instance.registeredOptions:# actually that can only happen if package is invalid CraftCore.log.error(f"{package} has no options") return False if key not in _instance.registeredOptions[package.path]: CraftCore.log.error(f"{package} unknown option {key}") CraftCore.log.error(f"Valid options are") for opt, default in _instance.registeredOptions[package.path].items(): default = default if callable(default) else type(default) CraftCore.log.error(f"\t{default.__name__} : {opt}") return False value = self._convert(_instance.registeredOptions[package.path][key], value) settings = _instance.initPackage(self) settings[key] = str(value) setattr(self, key, value) return True def registerOption(self, key : str, default, permanent=True) -> bool: _instance = UserOptions.instance() package = self._package if package.path not in _instance.registeredOptions: _instance.registeredOptions[package.path] = {} if key in _instance.registeredOptions[package.path]: raise BlueprintException(f"Failed to register option:\n[{package}]\n{key}={default}\nThe setting {key} is already registered.", package) return False _instance.registeredOptions[package.path][key] = default if permanent: settings = _instance.initPackage(self) if key and key not in settings: settings[key] = str(default) # don't try to save types if not callable(default): if not hasattr(self, key): setattr(self, key, default) else: # convert type old = getattr(self, key) try: new = self._convert(default, old) except: raise BlueprintException(f"Found an invalid option in BlueprintSettings.ini,\n[{self._package}]\n{key}={old}", self._package) #print(key, type(old), old, type(new), new) setattr(self, key, new) return True def __getattribute__(self, name): if name.startswith("_"): return super().__getattribute__(name) try: member = super().__getattribute__(name) except AttributeError: member = None if member and callable(member): return member #check cache _cache = super().__getattribute__("_cachedFromParent") if not member and name in _cache: return _cache[name] out = None _instance = UserOptions.instance() _package = super().__getattribute__("_package") _packagePath = _package.path if _packagePath in _instance.packageOptions and name in _instance.packageOptions[_packagePath]: if _packagePath not in _instance.registeredOptions or name not in _instance.registeredOptions[_packagePath]: raise BlueprintException(f"Package {_package} has no registered option {name}", _package) out = self._convert(_instance.registeredOptions[_packagePath][name], _instance.packageOptions[_packagePath][name]) elif member is not None: # value is not overwritten by comand line options return member else: parent = _package.parent if parent: out = getattr(UserOptions.get(parent), name) if not out: # name is a registered option and not a type but a default value if _packagePath in _instance.registeredOptions and name in _instance.registeredOptions[_packagePath]: default = _instance.registeredOptions[_packagePath][name] if not callable(default): out = default # skip lookup in command line options and parent objects the enxt time _cache[name] = out #print(_packagePath, name, type(out), out) return out class OptionsBase(object): def __init__(self): pass ## options for the fetch action class OptionsFetch(OptionsBase): def __init__(self): ## option comment self.option = None self.ignoreExternals = False ## enable submodule support in git single branch mode self.checkoutSubmodules = False ## options for the unpack action class OptionsUnpack(OptionsBase): def __init__(self): # Use this option to run 3rd party installers self.runInstaller = False ## options for the configure action class OptionsConfigure(OptionsBase): def __init__(self, dynamic): ## with this option additional arguments could be added to the configure commmand line self.args = dynamic.args ## with this option additional arguments could be added to the configure commmand line (for static builds) self.staticArgs = "" ## set source subdirectory as source root for the configuration tool. # Sometimes it is required to take a subdirectory from the source tree as source root # directory for the configure tool, which could be enabled by this option. The value of # this option is added to sourceDir() and the result is used as source root directory. self.configurePath = None # add the cmake defines that are needed to build tests here self.testDefine = None ## run autogen in autotools self.bootstrap = False # do not use default include path self.noDefaultInclude = False ## do not use default lib path self.noDefaultLib = False ## set this attribute in case a non standard configuration # tool is required (supported currently by QMakeBuildSystem only) self.tool = False # cflags currently only used for autotools self.cflags = "" # cxxflags currently only used for autotools self.cxxflags = "" # ldflags currently only used for autotools self.ldflags = "" # the project file, this is either a .pro for qmake or a sln for msbuild self.projectFile = None # whether to not pass --datarootdir configure self.noDataRootDir = False ## options for the make action class OptionsMake(OptionsBase): def __init__(self): ## ignore make error self.ignoreErrors = None ## options for the make tool self.args = "" self.supportsMultijob = True @property @deprecated("options.make.args") def makeOptions(self): return self.args @makeOptions.setter @deprecated("options.make.args") def makeOptions(self, x): self.args = x class OptionsInstall(OptionsBase): def __init__(self): ## options passed to make on install self.args = "install" ## options for the package action class OptionsPackage(OptionsBase): def __init__(self): ## defines the package name self.packageName = None ## defines the package version self.version = None ## use compiler in package name self.withCompiler = True ## use special packaging mode (only for qt) self.specialMode = False ## pack also sources self.packSources = True ## pack from subdir of imageDir() # currently supported by SevenZipPackager self.packageFromSubDir = None ## use architecture in package name # currently supported by SevenZipPackager self.withArchitecture = False ## add file digests to the package located in the manifest sub dir ##disable stripping of binary files # needed for mysql, striping make the library unusable self.disableStriping = False ##disable the binary cache for this package self.disableBinaryCache = False ## whether to move the plugins to bin self.movePluginsToBin = utils.OsUtils.isWin() ## main option class class Options(object): def __init__(self, package=None): self.dynamic = UserOptions.get(package) ## options of the fetch action self.fetch = OptionsFetch() ## options of the unpack action self.unpack = OptionsUnpack() ## options of the configure action self.configure = OptionsConfigure(self.dynamic) self.make = OptionsMake() self.install = OptionsInstall() ## options of the package action self.package = OptionsPackage() ## add the date to the target self.dailyUpdate = False ## has an issue with a too long path #enable by default for the ci # only applies for windows and if # [ShortPath] # EnableJunctions = True self.needsShortPath = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) ## there is a special option available already self.buildTools = False self.useShadowBuild = True @property def buildStatic(self): return self.dynamic.buildStatic def isActive(self, package): if isinstance(package, str): package = CraftPackageObject.get(package) return not package.isIgnored()