diff --git a/bin/CraftCommands.py b/bin/CraftCommands.py index 5503656b3..9e4b70d09 100644 --- a/bin/CraftCommands.py +++ b/bin/CraftCommands.py @@ -1,333 +1,333 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import tempfile import CraftBase from Blueprints.CraftDependencyPackage import CraftDependencyPackage, DependencyType from Blueprints.CraftVersion import CraftVersion from Blueprints.CraftPackageObject import CraftPackageObject from Utils.CraftTitleUpdater import CraftTitleUpdater from Utils import CraftTimer from options import * import glob import utils def doExec(package, action): with CraftTimer.Timer("%s for %s" % (action, package), 1): CraftCore.debug.step("Action: %s for %s" % (action, package)) ret = package.instance.execute(action) if not ret: if action == "fetch-binary": CraftCore.debug.step(f"{package} not found in cache") return False CraftCore.log.warning(f"Action: {action} for {package}:{package.version} FAILED") return ret def handlePackage(package, buildAction, directTargets): with CraftTimer.Timer(f"HandlePackage {package}", 3) as timer: success = True actions = [] timer.hook = lambda : utils.notify(f"Craft {buildAction} {'succeeded' if success else 'failed'}", f"{package} after {timer}", buildAction) CraftCore.debug.debug_line() CraftCore.debug.step(f"Handling package: {package}, action: {buildAction}") if buildAction == "all": if CraftCore.settings.getboolean("Packager", "UseCache", "False"): if doExec(package, "fetch-binary"): return True actions = ["fetch", "unpack", "compile", "cleanimage", "install", "post-install"] if CraftCore.settings.getboolean("ContinuousIntegration", "ClearBuildFolder", False): actions += ["cleanbuild"] actions += ["qmerge", "post-qmerge"] if CraftCore.settings.getboolean("Packager", "CreateCache"): onlyDirect = CraftCore.settings.getboolean("Packager", "CacheDirectTargetsOnly") if not onlyDirect or (onlyDirect and package in directTargets): actions += ["package"] else: actions = [buildAction] for action in actions: success = doExec(package, action) if not success: return False return True def resolvePackage(packageNames : [str], version : str=None) -> [CraftPackageObject]: package = CraftPackageObject(None) def resolveChildren(child): if child.isCategory(): for c in child.children.values(): resolveChildren(c) else: if version: UserOptions.addPackageOption(child, "version", version) package.children[child.name] = child for packageName in packageNames: child = CraftPackageObject.get(packageName) if not child: raise BlueprintNotFoundException(packageName) resolveChildren(child) return package def setOption(packageNames : [str], option : str) -> bool: if "=" not in option: CraftCore.log.error(f"Invalid option {option}") return False key, value = option.split("=", 1) for name in packageNames: package = CraftPackageObject.get(name) if not package: raise BlueprintNotFoundException(name) # create instance to make sure options are registered if not package.isCategory(): package.instance if not package: raise BlueprintNotFoundException(name) options = UserOptions.get(package) if not options.setOption(key, value): return False CraftCore.log.info(f"[{package}]\n{key}={getattr(options, key)}") return True def addBlueprintsRepository(url : str, args) -> bool: templateDir = os.path.join(CraftCore.standardDirs.craftBin(), "..", "internal_blueprints" ) with tempfile.TemporaryDirectory() as tmp: iniPath = os.path.join(tmp, "version.ini") parser = configparser.ConfigParser() parser.read(iniPath) parser.add_section("General") parser["General"]["branches"] = "master" parser["General"]["defaulttarget"] = "master" parser["General"]["gitUrl"] = url with open(iniPath, "wt+") as out: parser.write(out) CraftCore.settings.set("Blueprints", "Locations", templateDir) CraftCore.settings.set("InternalTemp", "add-bluprints-template.ini", iniPath) package = resolvePackage(["add-bluprints-template"]) return run(package, "fetch", args) def destroyCraftRoot() -> bool: settingsFiles = {"kdesettings.ini", "CraftSettings.ini", "BlueprintSettings.ini"} dirsToKeep = [CraftCore.standardDirs.downloadDir(), os.path.join(CraftCore.standardDirs.craftBin(), ".."), os.path.join(CraftCore.standardDirs.craftRoot(), "python"), CraftCore.standardDirs.blueprintRoot()] # dirs with possible interesting sub dirs maybeKeepDir = [ CraftCore.standardDirs.craftRoot(), CraftCore.standardDirs.etcDir(), os.path.join(CraftCore.standardDirs.etcDir(), "blueprints")# might contain blueprintRoot ] def deleteEntry(path): if utils.OsUtils.isLink(path): CraftCore.log.debug(f"Skipping symlink {path}") return if os.path.isdir(path): if any(os.path.exists(x) and os.path.samefile(path, x) for x in maybeKeepDir): CraftCore.log.debug(f"Path {path} in maybeKeepDir") for entry in os.listdir(path): deleteEntry(os.path.join(path, entry)) elif any(os.path.exists(x) and os.path.samefile(path, x) for x in dirsToKeep): CraftCore.log.debug(f"Path {path} in dirsToKeep") else: utils.cleanDirectory(path) utils.OsUtils.rmDir(path, True) else: if os.path.basename(path) not in settingsFiles: utils.OsUtils.rm(path, True) del CraftCore.installdb deleteEntry(CraftCore.standardDirs.craftRoot()) return True def readListFile(listFile): packageNames = [] parser = configparser.ConfigParser(allow_no_value=True) parser.read(listFile) for sections in parser.keys(): for packageName in parser[sections]: version = parser.get(sections, packageName) if version: UserOptions.setOptions([f"{packageName}.version={version}"]) packageNames.append(packageName) return packageNames def packageIsOutdated(package): installed = CraftCore.installdb.getInstalledPackages(package) if not installed: return True for pack in installed: version = pack.getVersion() if not version: continue cacheVersion = pack.getCacheVersion() if cacheVersion and cacheVersion != CraftBase.CraftBase.cacheVersion(): # can only happen for packages installed from cache return True return package.version != version def invoke(command : str, directTargets : [CraftPackageObject]) -> bool: args = {} key = command argsPattern = re.compile(r"(.+)\((.*)\)") argsMatch = argsPattern.findall(command) if argsMatch: key = argsMatch[0][0] args = eval(f"dict({','.join(argsMatch[0][1:])})") subs = key.split(".") for p in directTargets: instance = p.instance path = [] for sub in subs: path += [sub] if hasattr(instance, sub): attr = getattr(instance, sub) if callable(attr): instance = attr(**args) else: instance = attr else: CraftCore.debug.printOut(f"{p} has no member {'.'.join(path)}", file=sys.stderr) return False - CraftCore.log.debug(f"--get {command} on {p} -> {instance}") + CraftCore.log.debug(f"--get {command} on {p} -> {type(instance)} {instance}") CraftCore.debug.printOut(instance) return True def run(package : [CraftPackageObject], action : str, args) -> bool: if package.isIgnored(): CraftCore.log.info(f"Skipping package because it has been ignored: {package}") return True directTargets = package.children.values() CraftCore.state.directTargets = directTargets if action == "get": return invoke(args.get, directTargets) elif args.resolve_deps or action in ["all", "install-deps"]: # work on the dependencies depPackage = CraftDependencyPackage(package) if args.resolve_deps: if not args.resolve_deps.capitalize() in DependencyType.__members__: CraftCore.log.error(f"Invalid dependency type {args.resolve_deps}, valid types are {DependencyType.__members__}") return False depType = DependencyType.__getattr__(args.resolve_deps.capitalize()) print(depType) elif action == "install-deps": depType = DependencyType.Both else: depType = DependencyType.All depList = depPackage.getDependencies(depType=depType) packages = [] if not args.resolve_deps: for item in depList: if (args.ignoreInstalled and item in directTargets) or packageIsOutdated(item): packages.append(item) CraftCore.log.debug(f"dependency: {item}") else: packages = depList if not packages: CraftCore.log.debug("") if action == "install-deps": # we don't intend to build the package itself for x in directTargets: packages.remove(x) CraftTitleUpdater.usePackageProgressTitle(packages) while packages: info = packages[0] # in case we only want to see which packages are still to be build, simply return the package name if args.probe: CraftCore.log.warning(f"pretending {info}: {info.version}") else: if CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): CraftCore.debug.debug_line() CraftCore.log.info(f"Status: {CraftTitleUpdater.instance}") if action in ["install-deps"]: action = "all" if not handlePackage(info, action, directTargets=directTargets): CraftCore.log.error(f"fatal error: package {info} {action} failed") return False packages.pop(0) else: for info in directTargets: if not handlePackage(info, action, directTargets=directTargets): return False CraftCore.debug.new_line() return True def cleanBuildFiles(cleanArchives, cleanImages, cleanInstalledImages, cleanBuildDir, packages): def cleanDir(dir): CraftCore.debug.printOut(f"Cleaning: {dir}") utils.cleanDirectory(dir) os.rmdir(dir) for p in packages: package = CraftPackageObject.get(p.path) if not package or package.isCategory(): continue CraftCore.log.debug(f"Checking package for unused files: {p.path}") instance = package.instance version = instance.version if version: imageGlob = instance.imageDir().replace(version, "*") builddirGlob = instance.buildDir().replace(version, "*") else: imageGlob = instance.imageDir() builddirGlob = instance.buildDir() # image directories if cleanImages: for dir in glob.glob(imageGlob): if package.isInstalled and not cleanInstalledImages: if dir == instance.imageDir(): continue cleanDir(dir) # archive directory if cleanArchives and os.path.exists(instance.archiveDir()): cleanDir(instance.archiveDir()) # build directory if cleanBuildDir: for dir in glob.glob(builddirGlob): cleanDir(dir) def updateInstalled(args) -> bool: package = CraftPackageObject(None) for packageName, _ in CraftCore.installdb.getDistinctInstalled(): p = CraftPackageObject.get(packageName) if p: package.children[p.name] = p return run(package, "all", args) diff --git a/bin/options.py b/bin/options.py index f4797ee3a..529583753 100644 --- a/bin/options.py +++ b/bin/options.py @@ -1,442 +1,443 @@ ## @package property handling # # (c) copyright 2009-2011 Ralf Habacker # # # import utils from CraftConfig import * from CraftCore import CraftCore from Blueprints.CraftPackageObject import * from CraftDebug import deprecated import configparser import atexit import copy class UserOptions(object): class UserOptionsSingleton(object): _instance = None @property def __header(self): return """\ # The content of this file is partly autogenerated # You can modify values and add settings for your blueprints # Common settings available for all blueprints are: # ignored: [True|False] # version: some version # # use the same url as defined for the target but checks out a different branch # branch: str # patchLevel: int # buildType: [Release|RelWithDebInfo|Debug] The default is defined by CraftSettings.ini [Compile]BuildType # buildTests: [True|False] # buildStatic: [True|False] # # arguments passed to the configure step # args: str # # Example: ## [libs] ## ignored = True ## ## [lib/qt5] ## version = 5.9.3 ## ignored = False ## withMySQL = True ## ## [kde/pim/akonadi] ## args = -DAKONADI_BUILD_QSQLITE=On ## # # Settings are inherited, so you can set them for a whole sub branch or a single blueprint. # While blueprint from [libs] are all ignored blueprint from [libs/qt5] are not. # """ def __init__(self): self.cachedOptions = {} self.packageOptions = {} self.registeredOptions = {} self.path = CraftCore.settings.get("Blueprints", "Settings", os.path.join(CraftCore.standardDirs.etcDir(), "BlueprintSettings.ini")) self.settings = configparser.ConfigParser(allow_no_value=True) self.settings.optionxform = str if os.path.isfile(self.path): self.settings.read(self.path, encoding="utf-8") def initPackage(self, option): path = option._package.path if not self.settings.has_section(path): self.settings.add_section(path) settings = self.settings[path] return settings def toBool(self, x : str) -> bool: if not x: return False return self.settings._convert_to_boolean(x) @staticmethod @atexit.register def __dump(): instance = UserOptions.UserOptionsSingleton._instance if instance: try: with open(instance.path, "wt", encoding="utf-8") as configfile: print(instance.__header, file=configfile) instance.settings.write(configfile) except Exception as e: CraftCore.log.warning(f"Failed so save {instance.path}: {e}") @staticmethod def instance(): if not UserOptions.UserOptionsSingleton._instance: UserOptions.UserOptionsSingleton._instance = UserOptions.UserOptionsSingleton() return UserOptions.UserOptionsSingleton._instance def __init__(self, package): self._cachedFromParent = {} self._package = package _register = self.registerOption _convert = self._convert _register("version", str, permanent=False) _register("branch", str, permanent=False) _register("patchLevel", int, permanent=False) _register("ignored", bool, permanent=False) _register("buildTests", bool, permanent=False) _register("buildStatic",bool, permanent=False) _register("buildType", CraftCore.settings.get("Compile", "BuildType"), permanent=False) _register("args", "", permanent=False) settings = UserOptions.instance().settings if settings.has_section(package.path): _registered = UserOptions.instance().registeredOptions[package.path] for k, v in settings[package.path].items(): if k in _registered: v = _convert(_registered[k], v) setattr(self, k, v) @staticmethod def get(package): _instance = UserOptions.instance() packagePath = package.path if packagePath in _instance.cachedOptions: option = _instance.cachedOptions[packagePath] else: option = UserOptions(package) _instance.cachedOptions[packagePath] = option return option def _convert(self, valA, valB): """ Converts valB to type(valA) """ try: if valA is None: return valB _type = valA if callable(valA) else type(valA) if _type == type(valB): return valB if _type is bool: return UserOptions.instance().toBool(valB) return _type(valB) except Exception as e: CraftCore.log.error(f"Can't convert {valB} to {_type.__name__}") raise e @staticmethod def setOptions(optionsIn): packageOptions = UserOptions.instance().packageOptions sectionRe = re.compile(r"\[([^\[\]]+)\](.*)") for o in optionsIn: key, value = o.split("=", 1) key, value = key.strip(), value.strip() match = sectionRe.findall(key) if match: # TODO: move out of options.py section, key = match[0] CraftCore.log.info(f"setOptions: [{section}]{key} = {value}") CraftCore.settings.set(section, key, value) else: package, key = key.split(".", 1) if CraftPackageObject.get(package): if package not in packageOptions: packageOptions[package] = {} CraftCore.log.info(f"setOptions: BlueprintSettings.ini [{package}]{key} = {value}") packageOptions[package][key] = value else: raise BlueprintNotFoundException(package, f"Package {package} not found, failed to set option {key} = {value}") @staticmethod def addPackageOption(package : CraftPackageObject, key : str, value : str) -> None: if package.path not in UserOptions.instance().packageOptions: UserOptions.instance().packageOptions[package.path] = {} UserOptions.instance().packageOptions[package.path][key] = value def setOption(self, key, value) -> bool: _instance = UserOptions.instance() package = self._package if package.path not in _instance.registeredOptions:# actually that can only happen if package is invalid CraftCore.log.error(f"{package} has no options") return False if key not in _instance.registeredOptions[package.path]: CraftCore.log.error(f"{package} unknown option {key}") CraftCore.log.error(f"Valid options are") for opt, default in _instance.registeredOptions[package.path].items(): default = default if callable(default) else type(default) CraftCore.log.error(f"\t{default.__name__} : {opt}") return False settings = _instance.initPackage(self) if value == "" and key in settings: del settings[key] delattr(self, key) else: value = self._convert(_instance.registeredOptions[package.path][key], value) settings[key] = str(value) setattr(self, key, value) return True def registerOption(self, key : str, default, permanent=True, overwrite=False) -> bool: if not permanent and overwrite: CraftCore.log.error("An override must be permanent") return False _instance = UserOptions.instance() package = self._package if package.path not in _instance.registeredOptions: _instance.registeredOptions[package.path] = {} if not overwrite and key in _instance.registeredOptions[package.path]: raise BlueprintException(f"Failed to register option:\n[{package}]\n{key}={default}\nThe setting {key} is already registered.", package) return False _instance.registeredOptions[package.path][key] = default if permanent: settings = _instance.initPackage(self) if key and key not in settings: settings[key] = str(default) # don't try to save types if not callable(default): if not hasattr(self, key): setattr(self, key, default) else: # convert type old = getattr(self, key) try: - new = self._convert(old, default) + # convert the value read from the stting to the type of the now registed + new = self._convert(type(default), old) except: raise BlueprintException(f"Found an invalid option in BlueprintSettings.ini,\n[{self._package}]\n{key}={old}", self._package) #print(key, type(old), old, type(new), new) setattr(self, key, new) return True def __getattribute__(self, name): if name.startswith("_"): return super().__getattribute__(name) try: member = super().__getattribute__(name) except AttributeError: member = None if member and callable(member): return member #check cache _cache = super().__getattribute__("_cachedFromParent") if not member and name in _cache: return _cache[name] out = None _instance = UserOptions.instance() _package = super().__getattribute__("_package") _packagePath = _package.path if _packagePath in _instance.packageOptions and name in _instance.packageOptions[_packagePath]: if _packagePath not in _instance.registeredOptions or name not in _instance.registeredOptions[_packagePath]: raise BlueprintException(f"Package {_package} has no registered option {name}", _package) out = self._convert(_instance.registeredOptions[_packagePath][name], _instance.packageOptions[_packagePath][name]) elif member is not None: # value is not overwritten by comand line options return member else: parent = _package.parent if parent: out = getattr(UserOptions.get(parent), name) if not out: # name is a registered option and not a type but a default value if _packagePath in _instance.registeredOptions and name in _instance.registeredOptions[_packagePath]: default = _instance.registeredOptions[_packagePath][name] if not callable(default): out = default # skip lookup in command line options and parent objects the enxt time _cache[name] = out #print(_packagePath, name, type(out), out) return out class OptionsBase(object): def __init__(self): pass ## options for the fetch action class OptionsFetch(OptionsBase): def __init__(self): ## option comment self.option = None self.ignoreExternals = False ## enable submodule support in git single branch mode self.checkoutSubmodules = False ## options for the unpack action class OptionsUnpack(OptionsBase): def __init__(self): # Use this option to run 3rd party installers self.runInstaller = False ## options for the configure action class OptionsConfigure(OptionsBase): def __init__(self, dynamic): ## with this option additional arguments could be added to the configure commmand line self.args = dynamic.args ## with this option additional arguments could be added to the configure commmand line (for static builds) self.staticArgs = "" ## set source subdirectory as source root for the configuration tool. # Sometimes it is required to take a subdirectory from the source tree as source root # directory for the configure tool, which could be enabled by this option. The value of # this option is added to sourceDir() and the result is used as source root directory. self.configurePath = None # add the cmake defines that are needed to build tests here self.testDefine = None ## run autogen in autotools self.bootstrap = False ## run "autoreconf -vfi" in autotools self.autoreconf = True ## optional arguments for autoreconf self.autoreconfArgs = "-vfi" ## Whether to add the default -I flags when running autoreconf ## This is needed since some packages fail if we pass -I to autoreconf self.useDefaultAutoreconfIncludes = True # do not use default include path self.noDefaultInclude = False ## do not use default lib path self.noDefaultLib = False ## set this attribute in case a non standard configuration # tool is required (supported currently by QMakeBuildSystem only) self.tool = False # cflags currently only used for autotools self.cflags = "" # cxxflags currently only used for autotools self.cxxflags = "" # ldflags currently only used for autotools self.ldflags = "" # the project file, this is either a .pro for qmake or a sln for msbuild self.projectFile = None # whether to not pass --datarootdir configure self.noDataRootDir = False ## options for the make action class OptionsMake(OptionsBase): def __init__(self): ## ignore make error self.ignoreErrors = None ## options for the make tool self.args = "" self.supportsMultijob = True @property @deprecated("options.make.args") def makeOptions(self): return self.args @makeOptions.setter @deprecated("options.make.args") def makeOptions(self, x): self.args = x class OptionsInstall(OptionsBase): def __init__(self): ## options passed to make on install self.args = "install" ## options for the package action class OptionsPackage(OptionsBase): def __init__(self): ## defines the package name self.packageName = None ## defines the package version self.version = None ## use compiler in package name self.withCompiler = True ## use special packaging mode (only for qt) self.specialMode = False ## pack also sources self.packSources = True ## pack from subdir of imageDir() # currently supported by SevenZipPackager self.packageFromSubDir = None ## use architecture in package name # currently supported by SevenZipPackager self.withArchitecture = False ## add file digests to the package located in the manifest sub dir ##disable stripping of binary files # needed for mysql, striping make the library unusable self.disableStriping = False ##disable the binary cache for this package self.disableBinaryCache = False ## whether to move the plugins to bin self.movePluginsToBin = utils.OsUtils.isWin() ## main option class class Options(object): def __init__(self, package=None): self.dynamic = UserOptions.get(package) ## options of the fetch action self.fetch = OptionsFetch() ## options of the unpack action self.unpack = OptionsUnpack() ## options of the configure action self.configure = OptionsConfigure(self.dynamic) self.make = OptionsMake() self.install = OptionsInstall() ## options of the package action self.package = OptionsPackage() ## add the date to the target self.dailyUpdate = False ## there is a special option available already self.buildTools = False self.useShadowBuild = True @property def buildStatic(self): return self.dynamic.buildStatic def isActive(self, package): if isinstance(package, str): package = CraftPackageObject.get(package) return not package.isIgnored()