diff --git a/bin/Utils/CraftCache.py b/bin/Utils/CraftCache.py index d2b080b4c..315d152e5 100644 --- a/bin/Utils/CraftCache.py +++ b/bin/Utils/CraftCache.py @@ -1,199 +1,212 @@ import atexit import json import os import pickle import re import shutil import subprocess import time import urllib.error import urllib.request import sys from CraftCore import CraftCore, AutoImport import CraftDebug from Blueprints.CraftVersion import CraftVersion from CraftOS.unix.osutils import OsUtils from CraftStandardDirs import CraftStandardDirs class CraftCache(object): RE_TYPE = re.Pattern if sys.version_info >= (3,7) else re._pattern_type - _version = 8 + _version = 9 _cacheLifetime = (60 * 60 * 24) * 1 # days + class NonPersitenCache(object): + def __init__(self): + self.applicationLocations = {} + def __init__(self): self.version = CraftCache._version - self._appCache = {} + self.cacheCreationTime = time.time() + self._outputCache = {} self._helpCache = {} self._versionCache = {} self._nightlyVersions = {} - self.cacheCreationTime = time.time() + self._jsonCache = {} # defined in blueprintSearch self.availablePackages = None - self._jsonCache = {} + + # non persistent cache + self._nonPersitantCache = CraftCache.NonPersitenCache() + + def __getstate__(self): + state = dict(self.__dict__) + del state["_nonPersitantCache"] + return state @staticmethod def _loadInstance(): utilsCache = CraftCache() if os.path.exists(CraftCache._cacheFile()): with open(CraftCache._cacheFile(), "rb") as f: try: data = pickle.load(f) except: CraftCore.log.warning("Cache corrupted") return utilsCache if data.version != CraftCache._version or ( time.time() - data.cacheCreationTime) > CraftCache._cacheLifetime: CraftCore.log.debug("Clear cache") else: utilsCache = data + utilsCache._nonPersitantCache = CraftCache.NonPersitenCache() return utilsCache @staticmethod def _cacheFile(): return os.path.join(CraftStandardDirs.etcDir(), "cache.pickle") @staticmethod @atexit.register def _save(): try: if not os.path.isdir(os.path.dirname(CraftCache._cacheFile())): return if isinstance(CraftCore.cache, AutoImport): return with open(CraftCache._cacheFile(), "wb") as f: pick = pickle.Pickler(f, protocol=pickle.HIGHEST_PROTOCOL) pick.dump(CraftCore.cache) except Exception as e: CraftCore.log.warning(f"Failed to save cache {e}", exc_info=e, stack_info=True) os.remove(CraftCache._cacheFile()) def clear(self): CraftCore.log.debug("Clear utils cache") CraftCore.cache = CraftCache() def findApplication(self, app, path=None) -> str: - if app in self._appCache: - appLocation = self._appCache[app] + if app in self._nonPersitantCache.applicationLocations: + appLocation = self._nonPersitantCache.applicationLocations[app] if os.path.exists(appLocation): return appLocation else: self._helpCache.clear() appLocation = shutil.which(app, path=path) if appLocation: if OsUtils.isWin(): # prettify command path, ext = os.path.splitext(appLocation) appLocation = path + ext.lower() CraftCore.log.debug(f"Adding {app} to app cache {appLocation}") - self._appCache[app] = appLocation + self._nonPersitantCache.applicationLocations[app] = appLocation else: CraftCore.log.debug(f"Craft was unable to locate: {app}, in {path}") return None return appLocation def getCommandOutput(self, app:str, command:str, testName:str=None) -> (int, str): if not testName: testName = f"\"{app}\" {command}" app = self.findApplication(app) if not app: return (-1, None) if testName not in self._outputCache: CraftCore.log.debug(f"\"{app}\" {command}") # TODO: port away from shell=True completeProcess = subprocess.run(f"\"{app}\" {command}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, errors="backslashreplace") CraftCore.log.debug(f"{testName} Result: ExitedCode: {completeProcess.returncode} Output: {completeProcess.stdout}") self._outputCache[testName] = (completeProcess.returncode, completeProcess.stdout) return self._outputCache[testName] # TODO: rename, cleanup def checkCommandOutputFor(self, app, command, helpCommand="-h") -> str: if not (app, command) in self._helpCache: _, output = self.getCommandOutput(app, helpCommand) if not output: return False if type(command) == str: supports = command in output else: supports = command.match(output) is not None self._helpCache[(app, command)] = supports CraftCore.log.debug("%s %s %s" % (app, "supports" if supports else "does not support", command)) return self._helpCache[(app, command)] def getVersion(self, app, pattern=None, versionCommand=None) -> CraftVersion: if app in self._versionCache: return self._versionCache[app] app = self.findApplication(app) if not app: return None if not pattern: pattern = re.compile(r"(\d+\.\d+(?:\.\d+)?)") if not versionCommand: versionCommand = "--version" if not isinstance(pattern, CraftCache.RE_TYPE): raise Exception("getVersion can only handle a compiled regular expression as pattern") _, output = self.getCommandOutput(app, versionCommand) if not output: return False match = pattern.search(output) if not match: CraftCore.log.warning(f"Could not detect pattern: {pattern.pattern} in {output}") return False appVersion = CraftVersion(match.group(1)) self._versionCache[app] = appVersion return appVersion def cacheJsonFromUrl(self, url, timeout=10) -> object: CraftCore.log.debug(f"Fetch Json: {url}") if not url in self._jsonCache: if os.path.isfile(url): with open(url, "rt+") as jsonFile: # don't cache local manifest return json.loads(jsonFile.read()) else: try: with urllib.request.urlopen(url, timeout=timeout) as fh: jsonContent = str(fh.read(), "UTF-8") CraftCore.log.debug(f"Fetched json: {url}") CraftCore.log.debug(jsonContent) self._jsonCache[url] = json.loads(jsonContent) except urllib.error.HTTPError as e: CraftCore.log.debug(f"Failed to download {url}: {e}") if e.code == 404: # don't retry it self._jsonCache[url] = {} except Exception as e: CraftCore.log.debug(f"Failed to download {url}: {e}") return self._jsonCache.get(url, {}) def getNightlyVersionsFromUrl(self, url, pattern, timeout=10) -> [str]: """ Returns a list of possible version number matching the regular expression in pattern. :param url: The url to look for the nightly builds. :param pattern: A regular expression to match the version. :param timeout: :return: A list of matching strings or [None] """ if url not in self._nightlyVersions: if CraftCore.settings.getboolean("General", "WorkOffline"): CraftCore.debug.step("Nightly builds unavailable for %s in offline mode." % url) return [] try: with urllib.request.urlopen(url, timeout=timeout) as fh: data = str(fh.read(), "UTF-8") vers = re.findall(pattern, data) if not vers: print(data) raise Exception("Pattern %s does not match." % pattern) self._nightlyVersions[url] = list(set(vers)) return self._nightlyVersions[url] except Exception as e: CraftCore.log.warning("Nightly builds unavailable for %s: %s" % (url, e)) return self._nightlyVersions.get(url, []) diff --git a/blueprints/dev-utils/_windows/7zip/7zip.py b/blueprints/dev-utils/_windows/7zip/7zip.py index c8ee6bac5..d59a25968 100644 --- a/blueprints/dev-utils/_windows/7zip/7zip.py +++ b/blueprints/dev-utils/_windows/7zip/7zip.py @@ -1,43 +1,42 @@ import info from Package.MaybeVirtualPackageBase import * class subinfo(info.infoclass): def setTargets(self): for ver in ["1801"]: self.targets[ver] = f"https://files.kde.org/craft/3rdparty/7zip/7z{ver}-extra.zip" self.targetInstSrc[ver] = f"7z{ver}-extra" self.targetInstallPath[ver] = os.path.join("dev-utils", "bin") self.targetInstallPath["1604.1"] = os.path.join("dev-utils", "bin") self.targets["1604.1"] = f"https://files.kde.org/craft/3rdparty/7zip/7z1604-extra.zip" self.targetDigests["1604.1"] = (["350a20ec1a005255713c39911982bdeb091fc94ad9800448505c4772f8e85074"], CraftHash.HashAlgorithm.SHA256) self.targetDigests["1801"] = (["094c1120f3af512855e1c1df86c5f170cb649f2ca97a23834ccebc04ba575a7a"], CraftHash.HashAlgorithm.SHA256) self.description = "7-Zip is a file archiver with a high compression ratio." self.webpage = "http://www.7-zip.org/" self.defaultTarget = "1801" from Package.BinaryPackageBase import * class SevenZipPackage(BinaryPackageBase): def __init__(self): BinaryPackageBase.__init__(self) self.subinfo.options.package.disableBinaryCache = True def install(self): - CraftCore.cache.clear() if CraftCore.compiler.isX64(): return utils.copyFile(os.path.join(self.sourceDir(), "x64", "7za.exe"), os.path.join(self.installDir(), "7za.exe"), linkOnly=False) else: return utils.copyFile(os.path.join(self.sourceDir(), "7za.exe"), os.path.join(self.installDir(), "7za.exe"), linkOnly=False) class Package(VirtualIfSufficientVersion): def __init__(self): VirtualIfSufficientVersion.__init__(self, app="7za", version="16.04", versionCommand="-version", classA=SevenZipPackage) diff --git a/blueprints/dev-utils/_windows/python2/python2.py b/blueprints/dev-utils/_windows/python2/python2.py index ed6b9274c..39c5a7aea 100644 --- a/blueprints/dev-utils/_windows/python2/python2.py +++ b/blueprints/dev-utils/_windows/python2/python2.py @@ -1,30 +1,29 @@ import info from Package.BinaryPackageBase import * class subinfo(info.infoclass): def setTargets(self): self.targets["2"] = "" self.defaultTarget = "2" self.targetInstallPath["2"] = "dev-utils" class Package(BinaryPackageBase): def __init__(self): BinaryPackageBase.__init__(self) self.subinfo.options.package.disableBinaryCache = True def install(self): if not BinaryPackageBase.install(self): return False if not ("Paths", "PYTHON27") in CraftCore.settings or \ not os.path.isfile(os.path.join(CraftCore.settings.get("Paths", "PYTHON27"), "python.exe")): CraftCore.log.critical(f"Please have a look on {CraftCore.settings.iniPath} and make sure that\n" "\t[Paths]\n" "\tPYTHON27\n" "Points to a valid Python installation.") return False - CraftCore.cache.clear() return utils.createShim(os.path.join(self.installDir(), "bin", "python2.exe"), os.path.join(CraftCore.settings.get("Paths", "PYTHON27"), "python.exe"), useAbsolutePath=True) diff --git a/blueprints/dev-utils/_windows/python3/python3.py b/blueprints/dev-utils/_windows/python3/python3.py index 2c7a6e7cb..2a3499b59 100644 --- a/blueprints/dev-utils/_windows/python3/python3.py +++ b/blueprints/dev-utils/_windows/python3/python3.py @@ -1,27 +1,26 @@ import info from Package.BinaryPackageBase import * class subinfo(info.infoclass): def setTargets(self): self.targets["3"] = "" self.patchLevel["3"] = 1 self.targetInstallPath["3"] = "dev-utils" self.defaultTarget = "3" class Package(BinaryPackageBase): def __init__(self): BinaryPackageBase.__init__(self) self.subinfo.options.package.disableBinaryCache = True def install(self): if not BinaryPackageBase.install(self): return False - CraftCore.cache.clear() return (utils.createShim(os.path.join(self.installDir(), "bin", "python3.exe"), sys.executable, useAbsolutePath=True) and utils.createShim(os.path.join(self.installDir(), "bin", "python.exe"), sys.executable, useAbsolutePath=True))