diff --git a/bin/Source/GitSource.py b/bin/Source/GitSource.py index 17b65254a..f8bd2be86 100644 --- a/bin/Source/GitSource.py +++ b/bin/Source/GitSource.py @@ -1,217 +1,226 @@ # # copyright (c) 2009 Ralf Habacker # copyright (c) 2009 Patrick Spendrin # # git support import tempfile from Source.VersionSystemSourceBase import * ## \todo requires installed git package -> add suport for installing packages class GitSource(VersionSystemSourceBase): """git support""" def __init__(self, subinfo=None): CraftCore.debug.trace('GitSource __init__') if subinfo: self.subinfo = subinfo VersionSystemSourceBase.__init__(self) def __getCurrentBranch(self): branch = None if os.path.exists(self.checkoutDir()): tmpFile = tempfile.TemporaryFile() self.__git("branch", ["-a"], stdout=tmpFile) # TODO: check return value for success tmpFile.seek(0) for line in tmpFile: line = str(line, "UTF-8") if line.startswith("*"): branch = line[2:].rstrip() break return branch def __isLocalBranch(self, branch): if os.path.exists(self.checkoutDir()): tmpFile = tempfile.TemporaryFile() self.__git("branch", stdout=tmpFile) # TODO: check return value for success tmpFile.seek(0) for line in tmpFile: if str(line[2:].rstrip(), "UTF-8") == branch.rstrip(): return True return False def __isTag(self, _tag): if os.path.exists(self.checkoutDir()): tmpFile = tempfile.TemporaryFile() self.__git("tag", stdout=tmpFile) # TODO: check return value for success tmpFile.seek(0) for line in tmpFile: if str(line.rstrip(), "UTF-8") == _tag: return True return False def __getCurrentRevision(self): """return the revision returned by git show""" # run the command branch = self.__getCurrentBranch() if not self.__isTag(branch): # open a temporary file - do not use generic tmpfile because this doesn't give a good file object with python with tempfile.TemporaryFile() as tmpFile: self.__git("show", ["--abbrev-commit"], stdout=tmpFile) tmpFile.seek(os.SEEK_SET) # read the temporary file and grab the first line # print the revision - everything else should be quiet now line = tmpFile.readline() return "%s-%s" % (branch, str(line, "UTF-8").replace("commit ", "").strip()) else: # in case this is a tag, print out the tag version return branch def __git(self, command, args=None, logCommand=False, **kwargs): """executes a git command in a shell. Default for cwd is self.checkoutDir()""" parts = ["git", command] if command in ("clone", "checkout", "fetch", "pull", "submodule"): if CraftCore.debug.verbose() < 0: parts += ["-q"] else: kwargs["displayProgress"] = True else: kwargs["logCommand"] = logCommand if args: parts += args if not kwargs.get("cwd"): kwargs["cwd"] = self.checkoutDir() return utils.system(parts, **kwargs) def fetch(self): CraftCore.debug.trace('GitSource fetch') # get the path where the repositories should be stored to repopath = self.repositoryUrl() CraftCore.log.debug("fetching %s" % repopath) # in case you need to move from a read only Url to a writeable one, here it gets replaced repopath = repopath.replace("[git]", "") repoString = utils.replaceVCSUrl(repopath) [repoUrl, repoBranch, repoTag] = utils.splitVCSUrl(repoString) + # override tag + if self.subinfo.options.dynamic.revision: + repoTag = self.subinfo.options.dynamic.revision + repoBranch = "" + # override the branch if self.subinfo.options.dynamic.branch: repoBranch = self.subinfo.options.dynamic.branch + if repoTag and repoBranch: + CraftCore.log.error(f"Your not allowed to specify a branch and a tag: branch -> {repoBranch}, tag -> {repoTag}") + return False + if not repoBranch and not repoTag: repoBranch = "master" # only run if wanted (e.g. no --offline is given on the commandline) if (self.noFetch): CraftCore.log.debug("skipping git fetch (--offline)") return True else: ret = True checkoutDir = self.checkoutDir() # if we only have the checkoutdir but no .git within, # clean this up first if os.path.exists(checkoutDir) \ and not os.path.exists(os.path.join(checkoutDir, ".git")): os.rmdir(checkoutDir) if os.path.isdir(checkoutDir): if not repoTag: ret = self.__git("fetch") \ and self.__git("checkout", [repoBranch or "master"]) \ and self.__git("merge") if self.subinfo.options.fetch.checkoutSubmodules: ret = ret and self.__git("submodule", ["update", "--init", "--recursive"]) else: args = [] # it doesn't exist so clone the repo os.makedirs(checkoutDir) # first try to replace with a repo url from etc/blueprints/crafthosts.conf if self.subinfo.options.fetch.checkoutSubmodules: args += ["--recursive"] ret = self.__git('clone', args + [repoUrl, self.checkoutDir()]) # if a branch is given, we should check first if the branch is already downloaded # locally, otherwise we can track the remote branch if ret and repoBranch and not repoTag: if not self.__isLocalBranch(repoBranch): track = ["--track", f"origin/{repoBranch}"] else: track = [repoBranch] ret = self.__git("checkout", track) # we can have tags or revisions in repoTag if ret and repoTag: if self.__isTag(repoTag): if not self.__isLocalBranch("_" + repoTag): ret = self.__git('checkout', ['-b', f"_{repoTag}", repoTag]) else: ret = self.__git('checkout', [f"_{repoTag}"]) else: # unknown tag, try to fetch it first self.__git('fetch', ['--tags']) ret = self.__git('checkout', [repoTag]) return ret def applyPatch(self, fileName, patchdepth, unusedSrcDir=None): """apply single patch o git repository""" CraftCore.debug.trace('GitSource ') if fileName: patchfile = os.path.join(self.packageDir(), fileName) return self.__git('apply', ['--ignore-space-change', '-p', str(patchdepth), patchfile], logCommand=True) return True def createPatch(self): """create patch file from git source into the related package dir. The patch file is named autocreated.patch""" CraftCore.debug.trace('GitSource createPatch') patchFileName = os.path.join(self.packageDir(), "%s-%s.patch" % \ (self.package.name, str(datetime.date.today()).replace('-', ''))) CraftCore.log.debug("git diff %s" % patchFileName) with open(patchFileName, 'wt+') as patchFile: return self.__git('diff', stdout=patchFile) def sourceVersion(self): """print the revision returned by git show""" CraftCore.debug.trace('GitSource sourceVersion') return self.__getCurrentRevision() def checkoutDir(self, index=0): CraftCore.debug.trace('GitSource checkoutDir') return VersionSystemSourceBase.checkoutDir(self, index) def sourceDir(self, index=0): CraftCore.debug.trace('GitSource sourceDir') repopath = self.repositoryUrl() # in case you need to move from a read only Url to a writeable one, here it gets replaced repopath = repopath.replace("[git]", "") sourcedir = self.checkoutDir(index) if self.subinfo.hasTargetSourcePath(): sourcedir = os.path.join(sourcedir, self.subinfo.targetSourcePath()) CraftCore.log.debug("using sourcedir: %s" % sourcedir) parent, child = os.path.split(sourcedir) return os.path.join(CraftShortPath(parent).shortPath, child) def getUrls(self): """print the url where to clone from and the branch/tag/hash""" # in case you need to move from a read only Url to a writeable one, here it gets replaced repopath = self.repositoryUrl().replace("[git]", "") repoString = utils.replaceVCSUrl(repopath) [repoUrl, repoBranch, repoTag] = utils.splitVCSUrl(repoString) if not repoBranch and not repoTag: repoBranch = "master" print('|'.join([repoUrl, repoBranch, repoTag])) return True diff --git a/bin/options.py b/bin/options.py index 7066f5d01..da3e5baf7 100644 --- a/bin/options.py +++ b/bin/options.py @@ -1,465 +1,467 @@ ## @package property handling # # (c) copyright 2009-2011 Ralf Habacker # # # import utils from CraftConfig import * from CraftCore import CraftCore from Blueprints.CraftPackageObject import * from CraftDebug import deprecated import configparser import atexit import copy class UserOptions(object): class UserOptionsSingleton(object): _instance = None @property def __header(self): return """\ # The content of this file is partly autogenerated # You can modify values and add settings for your blueprints # Common settings available for all blueprints are: # ignored: [True|False] # version: some version # # use the same url as defined for the target but checks out a different branch -# branch: str +# branch: str A branch +# revision: str A revision or tag, overrides branch # patchLevel: int -# buildType: [Release|RelWithDebInfo|Debug] The default is defined by CraftSettings.ini [Compile]BuildType +# buildType: [Release|RelWithDebInfo|Debug] The default is defined by CraftSettings.ini [Compile]BuildType # buildTests: [True|False] # buildStatic: [True|False] # # arguments passed to the configure step # args: str # # Example: ## [libs] ## ignored = True ## ## [lib/qt5] ## version = 5.9.3 ## ignored = False ## withMySQL = True ## ## [kde/pim/akonadi] ## args = -DAKONADI_BUILD_QSQLITE=On ## # # Settings are inherited, so you can set them for a whole sub branch or a single blueprint. # While blueprint from [libs] are all ignored blueprint from [libs/qt5] are not. # """ def __init__(self): self.cachedOptions = {} self.packageOptions = {} self.registeredOptions = {} self.path = CraftCore.settings.get("Blueprints", "Settings", os.path.join(CraftCore.standardDirs.etcDir(), "BlueprintSettings.ini")) self.settings = configparser.ConfigParser(allow_no_value=True) self.settings.optionxform = str if os.path.isfile(self.path): self.settings.read(self.path, encoding="utf-8") def initPackage(self, option): path = option._package.path if not self.settings.has_section(path): self.settings.add_section(path) settings = self.settings[path] return settings def toBool(self, x : str) -> bool: if not x: return False return self.settings._convert_to_boolean(x) @staticmethod @atexit.register def __dump(): instance = UserOptions.UserOptionsSingleton._instance if instance: try: with open(instance.path, "wt", encoding="utf-8") as configfile: print(instance.__header, file=configfile) instance.settings.write(configfile) except Exception as e: CraftCore.log.warning(f"Failed so save {instance.path}: {e}") @staticmethod def instance(): if not UserOptions.UserOptionsSingleton._instance: UserOptions.UserOptionsSingleton._instance = UserOptions.UserOptionsSingleton() return UserOptions.UserOptionsSingleton._instance def __init__(self, package): self._cachedFromParent = {} self._package = package _register = self.registerOption _convert = self._convert _register("version", str, permanent=False) _register("branch", str, permanent=False) + _register("revision", str, permanent=False) _register("patchLevel", int, permanent=False) _register("ignored", bool, permanent=False) _register("buildTests", bool, permanent=False) _register("buildStatic",bool, permanent=False) _register("buildType", CraftCore.settings.get("Compile", "BuildType"), permanent=False) _register("args", "", permanent=False) settings = UserOptions.instance().settings if settings.has_section(package.path): _registered = UserOptions.instance().registeredOptions[package.path] for k, v in settings[package.path].items(): if k in _registered: v = _convert(_registered[k], v) setattr(self, k, v) def __str__(self): out = [] for k, v in UserOptions.instance().registeredOptions[self._package.path].items(): atr = getattr(self, k) if atr is None: if callable(v): atr = f"({v.__name__})" else: atr = v out.append((k, atr)) return ", ".join([f"{x}={y}" for x, y in sorted(out)]) @staticmethod def get(package): _instance = UserOptions.instance() packagePath = package.path if packagePath in _instance.cachedOptions: option = _instance.cachedOptions[packagePath] else: option = UserOptions(package) _instance.cachedOptions[packagePath] = option return option def _convert(self, valA, valB): """ Converts valB to type(valA) """ try: if valA is None: return valB _type = valA if callable(valA) else type(valA) if _type == type(valB): return valB if _type is bool: return UserOptions.instance().toBool(valB) return _type(valB) except Exception as e: CraftCore.log.error(f"Can't convert {valB} to {_type.__name__}") raise e @staticmethod def setOptions(optionsIn): packageOptions = UserOptions.instance().packageOptions sectionRe = re.compile(r"\[([^\[\]]+)\](.*)") for o in optionsIn: key, value = o.split("=", 1) key, value = key.strip(), value.strip() match = sectionRe.findall(key) if match: # TODO: move out of options.py section, key = match[0] CraftCore.log.info(f"setOptions: [{section}]{key} = {value}") CraftCore.settings.set(section, key, value) else: package, key = key.split(".", 1) if CraftPackageObject.get(package): if package not in packageOptions: packageOptions[package] = {} CraftCore.log.info(f"setOptions: BlueprintSettings.ini [{package}]{key} = {value}") packageOptions[package][key] = value else: raise BlueprintNotFoundException(package, f"Package {package} not found, failed to set option {key} = {value}") @staticmethod def addPackageOption(package : CraftPackageObject, key : str, value : str) -> None: if package.path not in UserOptions.instance().packageOptions: UserOptions.instance().packageOptions[package.path] = {} UserOptions.instance().packageOptions[package.path][key] = value def setOption(self, key, value) -> bool: _instance = UserOptions.instance() package = self._package if package.path not in _instance.registeredOptions:# actually that can only happen if package is invalid CraftCore.log.error(f"{package} has no options") return False if key not in _instance.registeredOptions[package.path]: CraftCore.log.error(f"{package} unknown option {key}") CraftCore.log.error(f"Valid options are") for opt, default in _instance.registeredOptions[package.path].items(): default = default if callable(default) else type(default) CraftCore.log.error(f"\t{default.__name__} : {opt}") return False settings = _instance.initPackage(self) if value == "" and key in settings: del settings[key] delattr(self, key) else: value = self._convert(_instance.registeredOptions[package.path][key], value) settings[key] = str(value) setattr(self, key, value) return True def registerOption(self, key : str, default, permanent=True) -> bool: _instance = UserOptions.instance() package = self._package if package.path not in _instance.registeredOptions: _instance.registeredOptions[package.path] = {} if key in _instance.registeredOptions[package.path]: raise BlueprintException(f"Failed to register option:\n[{package}]\n{key}={default}\nThe setting {key} is already registered.", package) return False _instance.registeredOptions[package.path][key] = default if permanent: settings = _instance.initPackage(self) if key and key not in settings: settings[key] = str(default) # don't try to save types if not callable(default): if not hasattr(self, key): setattr(self, key, default) else: # convert type old = getattr(self, key) try: new = self._convert(default, old) except: raise BlueprintException(f"Found an invalid option in BlueprintSettings.ini,\n[{self._package}]\n{key}={old}", self._package) #print(key, type(old), old, type(new), new) setattr(self, key, new) return True def setDefault(self, key : str, default) -> bool: _instance = UserOptions.instance() package = self._package if key not in _instance.registeredOptions[package.path]: raise BlueprintException(f"Failed to set default for unregistered option: [{package}]{key}.", package) return False settings = _instance.initPackage(self) _instance.registeredOptions[package.path][key] = default if key not in settings: settings[key] = str(default) setattr(self, key, default) return True def __getattribute__(self, name): if name.startswith("_"): return super().__getattribute__(name) try: member = super().__getattribute__(name) except AttributeError: member = None if member and callable(member): return member #check cache _cache = self._cachedFromParent if not member and name in _cache: return _cache[name] out = None _instance = UserOptions.instance() _package = self._package _packagePath = _package.path if _packagePath in _instance.packageOptions and name in _instance.packageOptions[_packagePath]: if _packagePath not in _instance.registeredOptions or name not in _instance.registeredOptions[_packagePath]: raise BlueprintException(f"Package {_package} has no registered option {name}", _package) out = self._convert(_instance.registeredOptions[_packagePath][name], _instance.packageOptions[_packagePath][name]) elif member is not None: # value is not overwritten by comand line options return member else: parent = _package.parent if parent: out = getattr(UserOptions.get(parent), name) if not out: # name is a registered option and not a type but a default value if _packagePath in _instance.registeredOptions and name in _instance.registeredOptions[_packagePath]: default = _instance.registeredOptions[_packagePath][name] if not callable(default): out = default # skip lookup in command line options and parent objects the enxt time _cache[name] = out #print(_packagePath, name, type(out), out) return out class OptionsBase(object): def __init__(self): pass ## options for the fetch action class OptionsFetch(OptionsBase): def __init__(self): ## option comment self.option = None self.ignoreExternals = False ## enable submodule support in git single branch mode self.checkoutSubmodules = False ## options for the unpack action class OptionsUnpack(OptionsBase): def __init__(self): # Use this option to run 3rd party installers self.runInstaller = False ## options for the configure action class OptionsConfigure(OptionsBase): def __init__(self, dynamic): ## with this option additional arguments could be added to the configure commmand line self.args = dynamic.args ## with this option additional arguments could be added to the configure commmand line (for static builds) self.staticArgs = "" ## set source subdirectory as source root for the configuration tool. # Sometimes it is required to take a subdirectory from the source tree as source root # directory for the configure tool, which could be enabled by this option. The value of # this option is added to sourceDir() and the result is used as source root directory. self.configurePath = None # add the cmake defines that are needed to build tests here self.testDefine = None ## run autogen in autotools self.bootstrap = False ## run "autoreconf -vfi" in autotools self.autoreconf = True ## optional arguments for autoreconf self.autoreconfArgs = "-vfi" ## Whether to add the default -I flags when running autoreconf ## This is needed since some packages fail if we pass -I to autoreconf self.useDefaultAutoreconfIncludes = True # do not use default include path self.noDefaultInclude = False ## do not use default lib path self.noDefaultLib = False ## set this attribute in case a non standard configuration # tool is required (supported currently by QMakeBuildSystem only) self.tool = False # cflags currently only used for autotools self.cflags = "" # cxxflags currently only used for autotools self.cxxflags = "" # ldflags currently only used for autotools self.ldflags = "" # the project file, this is either a .pro for qmake or a sln for msbuild self.projectFile = None # whether to not pass --datarootdir configure self.noDataRootDir = False ## options for the make action class OptionsMake(OptionsBase): def __init__(self): ## ignore make error self.ignoreErrors = None ## options for the make tool self.args = "" self.supportsMultijob = True @property @deprecated("options.make.args") def makeOptions(self): return self.args @makeOptions.setter @deprecated("options.make.args") def makeOptions(self, x): self.args = x class OptionsInstall(OptionsBase): def __init__(self): ## options passed to make on install self.args = "install" ## options for the package action class OptionsPackage(OptionsBase): def __init__(self): ## defines the package name self.packageName = None ## defines the package version self.version = None ## use compiler in package name self.withCompiler = True ## use special packaging mode (only for qt) self.specialMode = False ## pack also sources self.packSources = True ## pack from subdir of imageDir() # currently supported by SevenZipPackager self.packageFromSubDir = None ## use architecture in package name # currently supported by SevenZipPackager self.withArchitecture = False ## add file digests to the package located in the manifest sub dir ##disable stripping of binary files # needed for mysql, striping make the library unusable self.disableStriping = False ##disable the binary cache for this package self.disableBinaryCache = False ## whether to move the plugins to bin self.movePluginsToBin = utils.OsUtils.isWin() ## main option class class Options(object): def __init__(self, package=None): self.dynamic = UserOptions.get(package) ## options of the fetch action self.fetch = OptionsFetch() ## options of the unpack action self.unpack = OptionsUnpack() ## options of the configure action self.configure = OptionsConfigure(self.dynamic) self.make = OptionsMake() self.install = OptionsInstall() ## options of the package action self.package = OptionsPackage() ## add the date to the target self.dailyUpdate = False ## there is a special option available already self.buildTools = False self.useShadowBuild = True @property def buildStatic(self): return self.dynamic.buildStatic def isActive(self, package): if isinstance(package, str): package = CraftPackageObject.get(package) return not package.isIgnored()