diff --git a/bin/Package/PackageBase.py b/bin/Package/PackageBase.py --- a/bin/Package/PackageBase.py +++ b/bin/Package/PackageBase.py @@ -130,7 +130,6 @@ def fetchBinary(self, downloadRetriesLeft=3) -> bool: if self.subinfo.options.package.disableBinaryCache: return False - for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.") if url == self.cacheLocation(): @@ -147,12 +146,15 @@ for f in fileEntry: if f.version == self.version: files.append(f) - latest = None if not files: CraftCore.log.debug(f"Could not find {self}={self.version} in {url}") continue latest = files[0] + if latest.configHash and latest.configHash != self.subinfo.options.dynamic.configHash(): + CraftCore.log.warning("Failed to restore package, configuration missmatch") + return False + if url != self.cacheLocation(): downloadFolder = self.cacheLocation(os.path.join(CraftCore.standardDirs.downloadDir(), "cache")) else: diff --git a/bin/Packager/PackagerBase.py b/bin/Packager/PackagerBase.py --- a/bin/Packager/PackagerBase.py +++ b/bin/Packager/PackagerBase.py @@ -80,6 +80,6 @@ manifest = CraftManifest.load(manifestLocation, urls=manifestUrls) entry = manifest.get(str(self)) - entry.addFile(name, CraftHash.digestFile(archiveFile, CraftHash.HashAlgorithm.SHA256), version=self.version) + entry.addFile(name, CraftHash.digestFile(archiveFile, CraftHash.HashAlgorithm.SHA256), version=self.version, config=self.subinfo.options.dynamic) manifest.dump(manifestLocation) \ No newline at end of file diff --git a/bin/Utils/CraftManifest.py b/bin/Utils/CraftManifest.py --- a/bin/Utils/CraftManifest.py +++ b/bin/Utils/CraftManifest.py @@ -3,32 +3,35 @@ import json import os - from CraftCore import CraftCore -import utils class CraftManifestEntryFile(object): def __init__(self, fileName : str, checksum : str, version : str="") -> None: self.fileName = fileName self.checksum = checksum self.date = datetime.datetime.utcnow() self.version = version self.buildPrefix = CraftCore.standardDirs.craftRoot() + self.configHash = None @staticmethod def fromJson(data : dict): out = CraftManifestEntryFile(data["fileName"], data["checksum"]) out.date = CraftManifest._parseTimeStamp(data["date"]) out.version = data.get("version", "") out.buildPrefix = data.get("buildPrefix", None) + out.configHash = data.get("configHash", None) return out def toJson(self) -> dict: - return {"fileName" : self.fileName, - "checksum" : self.checksum, - "date" : self.date.strftime(CraftManifest._TIME_FORMAT), - "version" : self.version, - "buildPrefix" : self.buildPrefix} + return { + "fileName" : self.fileName, + "checksum" : self.checksum, + "date" : self.date.strftime(CraftManifest._TIME_FORMAT), + "version" : self.version, + "buildPrefix" : self.buildPrefix, + "configHash" : self.configHash + } class CraftManifestEntry(object): def __init__(self, name : str) -> None: @@ -44,8 +47,9 @@ def toJson(self) -> dict: return {"name":self.name, "files":[x.toJson() for x in self.files]} - def addFile(self, fileName : str, checksum : str, version : str="") -> CraftManifestEntryFile: + def addFile(self, fileName : str, checksum : str, version : str="", config=None) -> CraftManifestEntryFile: f = CraftManifestEntryFile(fileName, checksum, version) + f.configHash = config.configHash() self.files.insert(0, f) return f diff --git a/bin/options.py b/bin/options.py --- a/bin/options.py +++ b/bin/options.py @@ -4,15 +4,23 @@ # # # + import utils from CraftConfig import * from CraftCore import CraftCore from Blueprints.CraftPackageObject import * from CraftDebug import deprecated import configparser import atexit -import copy +import zlib +from typing import Dict + +class RegisteredOption(object): + def __init__(self, value, compatible): + self.value = value + # whether or not this change breaks binary cache compatibility + self.compatible = compatible class UserOptions(object): class UserOptionsSingleton(object): @@ -57,7 +65,7 @@ def __init__(self): self.cachedOptions = {} self.packageOptions = {} - self.registeredOptions = {} + self.registeredOptions = {} # type: Dict[str : RegisteredOption] self.path = CraftCore.settings.get("Blueprints", "Settings", os.path.join(CraftCore.standardDirs.etcDir(), "BlueprintSettings.ini")) @@ -111,32 +119,49 @@ _register("revision", str, permanent=False) _register("patchLevel", int, permanent=False) _register("ignored", bool, permanent=False) - _register("buildTests", bool, permanent=False) + _register("buildTests", bool, permanent=False, compatible=True) _register("buildStatic",bool, permanent=False) _register("buildType", CraftCore.settings.get("Compile", "BuildType"), permanent=False) _register("args", "", permanent=False) settings = UserOptions.instance().settings if settings.has_section(package.path): _registered = UserOptions.instance().registeredOptions[package.path] - for k, v in settings[package.path].items(): - if k in _registered: - v = _convert(_registered[k], v) - setattr(self, k, v) + for key, value in settings[package.path].items(): + if key in _registered: + value = _convert(_registered[key].value, value) + setattr(self, key, value) def __str__(self): out = [] - for k, v in UserOptions.instance().registeredOptions[self._package.path].items(): - atr = getattr(self, k) + for key, option in UserOptions.instance().registeredOptions[self._package.path].items(): + value = option.value + atr = getattr(self, key) if atr is None: - if callable(v): - atr = f"({v.__name__})" + if callable(value): + atr = f"({value.__name__})" else: - atr = v - out.append((k, atr)) + atr = value + out.append((key, atr)) return ", ".join([f"{x}={y}" for x, y in sorted(out)]) + def configHash(self): + tmp = [] + for key, option in sorted(UserOptions.instance().registeredOptions[self._package.path].items()): + # ignore flags that have no influence on the archive + if not option.compatible: + value = option.value + atr = getattr(self, key) + if atr is not None: + if key == "buildType": + # Releaseand and RelWithDebInfo are compatible + atr = 1 if atr in {"Release", "RelWithDebInfo"} else 0 + tmp.append(key.encode()) + tmp.append(bytes(atr, "UTF-8") if isinstance(atr, str) else bytes([atr])) + return zlib.adler32(b"".join(tmp)) + + @staticmethod def get(package): _instance = UserOptions.instance() @@ -196,55 +221,56 @@ def setOption(self, key, value) -> bool: - _instance = UserOptions.instance() + _instance = UserOptions.instance() # type: UserOptions.UserOptionsSingleton package = self._package if package.path not in _instance.registeredOptions:# actually that can only happen if package is invalid CraftCore.log.error(f"{package} has no options") return False if key not in _instance.registeredOptions[package.path]: CraftCore.log.error(f"{package} unknown option {key}") CraftCore.log.error(f"Valid options are") - for opt, default in _instance.registeredOptions[package.path].items(): + for optionKey, defaultOption in _instance.registeredOptions[package.path].items(): + default = defaultOption.value default = default if callable(default) else type(default) - CraftCore.log.error(f"\t{default.__name__} : {opt}") + CraftCore.log.error(f"\t{default.__name__} : {optionKey}") return False settings = _instance.initPackage(self) if value == "" and key in settings: del settings[key] delattr(self, key) else: - value = self._convert(_instance.registeredOptions[package.path][key], value) + value = self._convert(_instance.registeredOptions[package.path][key].value, value) settings[key] = str(value) setattr(self, key, value) return True - def registerOption(self, key : str, default, permanent=True) -> bool: + def registerOption(self, key : str, default, permanent=True, compatible=False) -> bool: _instance = UserOptions.instance() package = self._package if package.path not in _instance.registeredOptions: _instance.registeredOptions[package.path] = {} if key in _instance.registeredOptions[package.path]: raise BlueprintException(f"Failed to register option:\n[{package}]\n{key}={default}\nThe setting {key} is already registered.", package) return False - _instance.registeredOptions[package.path][key] = default + _instance.registeredOptions[package.path][key] = RegisteredOption(default, compatible) if permanent: settings = _instance.initPackage(self) if key and key not in settings: settings[key] = str(default) - # don't try to save types - if not callable(default): - if not hasattr(self, key): - setattr(self, key, default) - else: - # convert type - old = getattr(self, key) - try: - new = self._convert(default, old) - except: - raise BlueprintException(f"Found an invalid option in BlueprintSettings.ini,\n[{self._package}]\n{key}={old}", self._package) - #print(key, type(old), old, type(new), new) - setattr(self, key, new) + # don't try to save types + if not callable(default): + if not hasattr(self, key): + setattr(self, key, default) + else: + # convert type + old = getattr(self, key) + try: + new = self._convert(default, old) + except: + raise BlueprintException(f"Found an invalid option in BlueprintSettings.ini,\n[{self._package}]\n{key}={old}", self._package) + #print(key, type(old), old, type(new), new) + setattr(self, key, new) return True def setDefault(self, key : str, default) -> bool: @@ -255,7 +281,7 @@ return False settings = _instance.initPackage(self) - _instance.registeredOptions[package.path][key] = default + _instance.registeredOptions[package.path][key].value = default if key not in settings: settings[key] = str(default) setattr(self, key, default) @@ -284,7 +310,7 @@ if _packagePath in _instance.packageOptions and name in _instance.packageOptions[_packagePath]: if _packagePath not in _instance.registeredOptions or name not in _instance.registeredOptions[_packagePath]: raise BlueprintException(f"Package {_package} has no registered option {name}", _package) - out = self._convert(_instance.registeredOptions[_packagePath][name], _instance.packageOptions[_packagePath][name]) + out = self._convert(_instance.registeredOptions[_packagePath][name].value, _instance.packageOptions[_packagePath][name]) elif member is not None: # value is not overwritten by comand line options return member @@ -296,7 +322,7 @@ if not out: # name is a registered option and not a type but a default value if _packagePath in _instance.registeredOptions and name in _instance.registeredOptions[_packagePath]: - default = _instance.registeredOptions[_packagePath][name] + default = _instance.registeredOptions[_packagePath][name].value if not callable(default): out = default