diff --git a/bin/Source/ArchiveSource.py b/bin/Source/ArchiveSource.py index 2b00ddea4..73bee5941 100644 --- a/bin/Source/ArchiveSource.py +++ b/bin/Source/ArchiveSource.py @@ -1,312 +1,314 @@ # -*- coding: utf-8 -*- # copyright (c) 2009 Ralf Habacker # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import tempfile import io from pathlib import Path +import urllib from Source.SourceBase import * from Utils import CraftHash, GetFiles, CraftChoicePrompt from Utils.CraftManifest import CraftManifest from CraftCore import CraftCore class ArchiveSource(SourceBase): """ file download source""" def __init__(self): CraftCore.log.debug("ArchiveSource.__init__ called") SourceBase.__init__(self) self.__archiveDir = Path(CraftCore.standardDirs.downloadDir()) / "archives" self.__downloadDir = self.__archiveDir / self.package.path def _generateSrcManifest(self, archiveNames): if archiveNames == [""]: return True manifestLocation = os.path.join(self.__archiveDir, "manifest.json") manifest = CraftManifest.load(manifestLocation, urls=CraftCore.settings.getList("Packager", "ArchiveRepositoryUrl")) entry = manifest.get(str(self)) entry.files.clear() for archiveName in archiveNames: name = (Path(self.package.path) / archiveName).as_posix() archiveFile = self.__downloadDir / archiveName if not archiveFile.is_file(): continue digests = CraftHash.digestFile(archiveFile, CraftHash.HashAlgorithm.SHA256) entry.addFile(name, digests, version=self.version) manifest.dump(manifestLocation) return True def localFileNames(self): if self.subinfo.archiveName() == [""]: return self.localFileNamesBase() if isinstance(self.subinfo.archiveName(), (tuple, list)): return self.subinfo.archiveName() return [self.subinfo.archiveName()] def localFilePath(self): return [os.path.join(self.__downloadDir, f) for f in self.localFileNamesBase()] def localFileNamesBase(self): """ collect local filenames """ CraftCore.log.debug("ArchiveSource.localFileNamesBase called") filenames = [] for url in self.subinfo.targets.values(): filenames.append(os.path.basename(url)) return filenames def __checkFilesPresent(self, filenames): def isFileValid(path): if not os.path.exists(path): return False return os.path.getsize(path) > 0 """check if all files for the current target are available""" for filename in filenames: path = os.path.join(self.__downloadDir, filename) # check file if not isFileValid(path): return False # check digests if self.subinfo.hasTargetDigests(): if not isFileValid(path): return False elif self.subinfo.hasTargetDigestUrls(): algorithm = CraftHash.HashAlgorithm.SHA1 if type(self.subinfo.targetDigestUrl()) == tuple: _, algorithm = self.subinfo.targetDigestUrl() if not isFileValid(path + algorithm.fileEnding()): return False return True def fetch(self, downloadRetriesLeft=3): """fetch normal tarballs""" CraftCore.log.debug("ArchiveSource.fetch called") filenames = self.localFileNames() if (self.noFetch): CraftCore.log.debug("skipping fetch (--offline)") return True if self.subinfo.hasTarget(): if self.__checkFilesPresent(filenames): CraftCore.log.debug("files and digests available, no need to download files") return True if self.subinfo.target(): + for url in CraftCore.settings.getList("Packager", "ArchiveRepositoryUrl"): - manifest = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) + manifest = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(urllib.parse.urljoin(url, "manifest.json"))) files = manifest.get(str(self)).files if files: self.__downloadDir.mkdir(parents=True, exist_ok=True) for entry in files: - if not GetFiles.getFile(f"{url}{entry.fileName}", self.__archiveDir, entry.fileName): + if not GetFiles.getFile(urllib.parse.urljoin(url, entry.fileName), self.__archiveDir, entry.fileName): return False if not CraftHash.checkFilesDigests(self.__archiveDir, [entry.fileName], digests=entry.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): return False if self.__checkFilesPresent(filenames): CraftCore.log.debug("files and digests available, no need to download files") return True break # compat for scripts that provide multiple files files = zip(self.subinfo.target(), self.subinfo.archiveName()) if isinstance(self.subinfo.target(), list) else [(self.subinfo.target(), self.subinfo.archiveName()[0])] for url, fileName in files: if not GetFiles.getFile(url, self.__downloadDir, fileName): CraftCore.log.debug("failed to download files") return False if self.subinfo.hasTargetDigestUrls(): if isinstance(self.subinfo.targetDigestUrl(), tuple): url, alg = self.subinfo.targetDigestUrl() return GetFiles.getFile(url[0], self.__downloadDir, self.subinfo.archiveName()[0] + CraftHash.HashAlgorithm.fileEndings().get(alg)) else: for url in self.subinfo.targetDigestUrl(): if not GetFiles.getFile(url, self.__downloadDir): return False else: CraftCore.log.debug("no digestUrls present") if downloadRetriesLeft and not self.__checkFilesPresent(filenames): return ArchiveSource.fetch(self, downloadRetriesLeft=downloadRetriesLeft - 1) return True def checkDigest(self, downloadRetriesLeft=3): CraftCore.log.debug("ArchiveSource.checkDigest called") filenames = self.localFileNames() if self.subinfo.hasTargetDigestUrls(): CraftCore.log.debug("check digests urls") if not CraftHash.checkFilesDigests(self.__downloadDir, filenames): CraftCore.log.error("invalid digest file") redownload = downloadRetriesLeft and CraftChoicePrompt.promptForChoice("Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes") if redownload: for filename in filenames: CraftCore.log.info(f"Deleting downloaded file: {filename}") utils.deleteFile(os.path.join(self.__downloadDir, filename)) for digestAlgorithm, digestFileEnding in CraftHash.HashAlgorithm.fileEndings().items(): digestFileName = filename + digestFileEnding if os.path.exists(os.path.join(self.__downloadDir, digestFileName)): CraftCore.log.info(f"Deleting downloaded file: {digestFileName}") utils.deleteFile(os.path.join(self.__downloadDir, digestFileName)) return self.fetch() and self.checkDigest(downloadRetriesLeft - 1) return False elif self.subinfo.hasTargetDigests(): CraftCore.log.debug("check digests") digests, algorithm = self.subinfo.targetDigest() if not CraftHash.checkFilesDigests(self.__downloadDir, filenames, digests, algorithm): CraftCore.log.error("invalid digest file") redownload = downloadRetriesLeft and CraftChoicePrompt.promptForChoice("Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes") if redownload: for filename in filenames: CraftCore.log.info(f"Deleting downloaded file: {filename}") utils.deleteFile(os.path.join(self.__downloadDir, filename)) return self.fetch() and self.checkDigest(downloadRetriesLeft - 1) return False else: CraftCore.log.debug("print source file digests") CraftHash.printFilesDigests(self.__downloadDir, filenames, self.subinfo.buildTarget, algorithm=CraftHash.HashAlgorithm.SHA256) if self.subinfo.hasTargetDigestUrls(): url, alg = self.subinfo.targetDigestUrl() filenames.append(self.subinfo.archiveName()[0] + CraftHash.HashAlgorithm.fileEndings().get(alg)) return self._generateSrcManifest(filenames) def unpack(self): """unpacking all zipped(gz, zip, bz2) tarballs""" CraftCore.log.debug("ArchiveSource.unpack called") filenames = self.localFileNames() # TODO: this might delete generated patches utils.cleanDirectory(self.workDir()) if not self.checkDigest(3): return False binEndings = (".exe", ".bat", ".msi") for filename in filenames: if filename.endswith(binEndings): filePath = os.path.abspath(os.path.join(self.__downloadDir, filename)) if self.subinfo.options.unpack.runInstaller: _, ext = os.path.splitext(filename) if ext == ".exe": return utils.system("%s %s" % (filePath, self.subinfo.options.configure.args)) elif (ext == ".msi"): return utils.system("msiexec /package %s %s" % (filePath, self.subinfo.options.configure.args)) if not utils.copyFile(filePath, os.path.join(self.workDir(), filename)): return False else: if not utils.unpackFile(self.__downloadDir, filename, self.workDir()): return False ret = self.applyPatches() if CraftCore.settings.getboolean("General", "EMERGE_HOLD_ON_PATCH_FAIL", False): return ret return True def getUrls(self): print(self.subinfo.target()) print(self.subinfo.targetDigestUrl()) return True def createPatch(self): """ unpacking all zipped(gz, zip, bz2) tarballs a second time and making a patch """ if not CraftCore.cache.findApplication("diff"): CraftCore.log.critical("could not find diff tool, please run 'craft diffutils'") return False # get the file paths of the tarballs filenames = self.localFileNames() destdir = self.workDir() # it makes no sense to make a diff against nothing if (not os.path.exists(self.sourceDir())): CraftCore.log.error("source directory doesn't exist, please run unpack first") return False CraftCore.log.debug("unpacking files into work root %s" % destdir) # make a temporary directory so the original packages don't overwrite the already existing ones with tempfile.TemporaryDirectory() as tmpdir: _patchName = f"{self.package.name}-{self.buildTarget}-{str(datetime.date.today()).replace('-', '')}.diff" # unpack all packages for filename in filenames: CraftCore.log.debug(f"unpacking this file: {filename}") if (not utils.unpackFile(self.__downloadDir, filename, tmpdir)): return False patches = self.subinfo.patchesToApply() if not isinstance(patches, list): patches = [patches] for fileName, patchdepth in patches: if os.path.basename(fileName) == _patchName: CraftCore.log.info(f"skipping patch {fileName} with patchlevel: {patchdepth}") continue CraftCore.log.info(f"applying patch {fileName} with patchlevel: {patchdepth}") if not self.applyPatch(fileName, patchdepth, os.path.join(tmpdir, os.path.relpath(self.sourceDir(), self.workDir()))): return False srcSubDir = os.path.relpath(self.sourceDir(), self.workDir()) tmpSourceDir = os.path.join(tmpdir, srcSubDir) with io.BytesIO() as out: ignores = [] for x in ["*~", r"*\.rej", r"*\.orig", r"*\.o", r"*\.pyc", "CMakeLists.txt.user"]: ignores += ["-x", x] # TODO: actually we should not accept code 2 if not utils.system(["diff", "-Nrub"] + ignores + [tmpSourceDir, self.sourceDir()], stdout=out, acceptableExitCodes=[0,1,2], cwd=destdir): return False patchContent = out.getvalue() # make the patch a -p1 patch patchContent = patchContent.replace(tmpSourceDir.encode(), f"{srcSubDir}.orig".encode()) patchContent = patchContent.replace(self.sourceDir().encode(), srcSubDir.encode()) patchPath = os.path.join(self.packageDir(), _patchName) with open(patchPath, "wb") as out: out.write(patchContent) CraftCore.log.info(f"Patch created {patchPath} self.patchToApply[\"{self.buildTarget}\"] = [(\"{_patchName}\", 1)]") return True def sourceVersion(self): """ return a version based on the file name of the current target """ # we hope that the build target is equal to the version that is build return self.subinfo.buildTarget diff --git a/bin/Utils/CraftManifest.py b/bin/Utils/CraftManifest.py index b97fd8c04..a66b633d5 100644 --- a/bin/Utils/CraftManifest.py +++ b/bin/Utils/CraftManifest.py @@ -1,184 +1,185 @@ import collections import datetime import json import os import shutil from pathlib import Path from typing import List +import urllib from CraftCore import CraftCore class CraftManifestEntryFile(object): def __init__(self, fileName : str, checksum : str, version : str="") -> None: self.fileName = fileName self.checksum = checksum self.date = datetime.datetime.utcnow() self.version = version self.buildPrefix = CraftCore.standardDirs.craftRoot() self.configHash = None if CraftCore.compiler.isWindows: self.fileName = self.fileName.replace("\\", "/") @staticmethod def fromJson(data : dict): out = CraftManifestEntryFile(data["fileName"], data["checksum"]) out.date = CraftManifest._parseTimeStamp(data["date"]) out.version = data.get("version", "") out.buildPrefix = data.get("buildPrefix", None) out.configHash = data.get("configHash", None) return out def toJson(self) -> dict: data = { "fileName" : self.fileName, "checksum" : self.checksum, "date" : self.date.strftime(CraftManifest._TIME_FORMAT), "version" : self.version } if self.configHash: data.update({ "buildPrefix" : self.buildPrefix, "configHash" : self.configHash }) return data class CraftManifestEntry(object): def __init__(self, name : str) -> None: self.name = name self.files = [] # type: List[CraftManifestEntryFile] @staticmethod def fromJson(data : dict): entry = CraftManifestEntry(data["name"]) entry.files = sorted([CraftManifestEntryFile.fromJson(fileData) for fileData in data["files"]], key=lambda x:x.date, reverse=True) return entry def toJson(self) -> dict: return {"name":self.name, "files": [x.toJson() for x in collections.OrderedDict.fromkeys(self.files)]} def addFile(self, fileName : str, checksum : str, version : str="", config=None) -> CraftManifestEntryFile: f = CraftManifestEntryFile(fileName, checksum, version) if config: f.configHash = config.configHash() self.files.insert(0, f) return f @property def latest(self) -> CraftManifestEntryFile: return self.files[0] if self.files else None class CraftManifest(object): _TIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f" def __init__(self): self.date = datetime.datetime.utcnow() self.packages = {str(CraftCore.compiler) : {}} self.origin = None @staticmethod def version() -> int: return 1 @staticmethod def _migrate0(data : dict): manifest = CraftManifest() packages = manifest.packages[str(CraftCore.compiler)] for name, package in data.items(): if not name in packages: packages[name] = CraftManifestEntry(name) p = packages[name] for fileName, pData in data[name].items(): f = p.addFile(fileName, pData["checksum"]) f.date = datetime.datetime(1, 1, 1) return manifest @staticmethod def fromJson(data : dict): version = data.get("version", 0) if version == 0: return CraftManifest._migrate0(data) elif version != CraftManifest.version(): raise Exception("Invalid manifest version detected") manifest = CraftManifest() manifest.date = CraftManifest._parseTimeStamp(data["date"]) manifest.origin = data.get("origin", None) for compiler in data["packages"]: manifest.packages[compiler] = {} for package in data["packages"][compiler]: p = CraftManifestEntry.fromJson(package) manifest.packages[compiler][p.name] = p return manifest def update(self, other): for compiler in other.packages.keys(): if not compiler in self.packages: self.packages[compiler] = {} self.packages[compiler].update(other.packages[compiler]) def toJson(self) -> dict: out = {"date": str(self.date), "origin": self.origin, "packages":{}, "version": CraftManifest.version()} for compiler, packages in self.packages.items(): out["packages"][compiler] = [x.toJson() for x in self.packages[compiler].values()] return out def get(self, package : str) -> CraftManifestEntry: compiler = str(CraftCore.compiler) if not compiler in self.packages: self.packages[compiler] = {} if not package in self.packages[compiler]: self.packages[compiler][package] = CraftManifestEntry(package) return self.packages[compiler][package] def dump(self, cacheFilePath): cacheFilePath = Path(cacheFilePath) cacheFilePathTimed = cacheFilePath.parent / f"{cacheFilePath.stem}-{self.date.strftime('%Y%m%dT%H%M%S')}{cacheFilePath.suffix}" self.date = datetime.datetime.utcnow() if self.origin: CraftCore.log.info(f"Updating cache manifest from: {self.origin} in: {cacheFilePath}") else: CraftCore.log.info(f"Create new cache manifest: {cacheFilePath}") with open(cacheFilePath, "wt") as cacheFile: json.dump(self, cacheFile, sort_keys=True, indent=2, default=lambda x:x.toJson()) shutil.copy2(cacheFilePath, cacheFilePathTimed) @staticmethod def load(manifestFileName : str, urls : [str]=None): """ Load a manifest. If a url is provided a manifest is fetch from that the url and merged with a local manifest. TODO: in that case we are merging all repositories so we should also merge the cache files """ old = None if not urls and ("ContinuousIntegration", "RepositoryUrl") in CraftCore.settings: urls = [CraftCore.settings.get("ContinuousIntegration", "RepositoryUrl").rstrip("/")] if urls: old = CraftManifest() for url in urls: - new = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) + new = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(urllib.parse.urljoin(url, "manifest.json"))) if new: new.origin = url old.update(new) cache = None if os.path.isfile(manifestFileName): try: with open(manifestFileName, "rt") as cacheFile: cache = CraftManifest.fromJson(json.load(cacheFile)) except Exception as e: CraftCore.log.warning(f"Failed to load {cacheFile}, {e}") pass if old: if cache: old.update(cache) return old if not cache: return CraftManifest() return cache @staticmethod def _parseTimeStamp(time : str) -> datetime.datetime: return datetime.datetime.strptime(time, CraftManifest._TIME_FORMAT)