diff --git a/bin/Package/PackageBase.py b/bin/Package/PackageBase.py index 8c301182d..e32b4dacd 100644 --- a/bin/Package/PackageBase.py +++ b/bin/Package/PackageBase.py @@ -1,229 +1,228 @@ # # copyright (c) 2009 Ralf Habacker # from pathlib import Path from CraftBase import * from CraftCompiler import * from InstallDB import * from Blueprints.CraftPackageObject import * from Utils import CraftHash, GetFiles, CraftChoicePrompt from Utils.CraftManifest import CraftManifest import json class PackageBase(CraftBase): """ provides a generic interface for packages and implements the basic stuff for all packages """ # uses the following instance variables # todo: place in related ...Base # rootdir -> CraftBase # package -> PackageBase # force -> PackageBase # category -> PackageBase # version -> PackageBase # packagedir -> PackageBase # imagedir -> PackageBase def __init__(self): CraftCore.log.debug("PackageBase.__init__ called") CraftBase.__init__(self) def qmerge(self): """mergeing the imagedirectory into the filesystem""" ## \todo is this the optimal place for creating the post install scripts ? if self.package.isInstalled: self.unmerge() copiedFiles = [] # will be populated by the next call if not utils.copyDir(self.imageDir(), CraftCore.standardDirs.craftRoot(), copiedFiles=copiedFiles): return False # add package to installed database -> is this not the task of the manifest files ? revision = self.sourceRevision() package = CraftCore.installdb.addInstalled(self.package, self.version, revision=revision) fileList = self.getFileListFromDirectory(CraftCore.standardDirs.craftRoot(), copiedFiles) package.addFiles(fileList) package.install() if (CraftCore.settings.getboolean("Packager", "CreateCache") or CraftCore.settings.getboolean("Packager", "UseCache")): package.setCacheVersion(self.cacheVersion()) return True def unmerge(self): """unmergeing the files from the filesystem""" CraftCore.log.debug("Packagebase unmerge called") packageList = CraftCore.installdb.getInstalledPackages(self.package) for package in packageList: fileList = package.getFilesWithHashes() self.unmergeFileList(CraftCore.standardDirs.craftRoot(), fileList) package.uninstall() return True def createImportLibs(self, pkgName): """create the import libraries for the other compiler(if ANSI-C libs)""" basepath = os.path.join(self.installDir()) utils.createImportLibs(pkgName, basepath) def fetchBinary(self, downloadRetriesLeft=3) -> bool: if self.subinfo.options.package.disableBinaryCache: return False for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.") if url == self.cacheLocation(): fileUrl = f"{url}/manifest.json" if os.path.exists(fileUrl): with open(fileUrl, "rt", encoding="UTF-8") as f: manifest = CraftManifest.fromJson(json.load(f)) else: continue else: manifest = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) fileEntry = manifest.get(str(self)).files files = [] for f in fileEntry: if f.version == self.version: files.append(f) if not files: CraftCore.log.debug(f"Could not find {self}={self.version} in {url}") continue latest = files[0] if latest.configHash and latest.configHash != self.subinfo.options.dynamic.configHash(): CraftCore.log.warning("Failed to restore package, configuration missmatch") # try next cache continue # if we are creating the cache, a rebuild on a failed fetch would be suboptimal createingCache = CraftCore.settings.getboolean("Packager", "CreateCache", False) if url != self.cacheLocation(): downloadFolder = self.cacheLocation(os.path.join(CraftCore.standardDirs.downloadDir(), "cache")) else: downloadFolder = self.cacheLocation() localArchiveAbsPath = OsUtils.toNativePath(os.path.join(downloadFolder, latest.fileName)) localArchivePath, localArchiveName = os.path.split(localArchiveAbsPath) if url != self.cacheLocation(): if not os.path.exists(localArchiveAbsPath): os.makedirs(localArchivePath, exist_ok=True) fileName = latest.fileName if CraftCore.compiler.isWindows: fileName = fileName.replace("\\", "/") fUrl = f"{url}/{fileName}" if not GetFiles.getFile(fUrl, localArchivePath, localArchiveName): msg = f"Failed to fetch {fUrl}" if createingCache: raise BlueprintException(msg, self.package) else: CraftCore.log.warning(msg) return False elif not os.path.isfile(localArchiveAbsPath): continue if not CraftHash.checkFilesDigests(localArchivePath, [localArchiveName], digests=latest.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): msg = f"Hash did not match, {localArchiveName} might be corrupted" CraftCore.log.warning(msg) if downloadRetriesLeft and CraftChoicePrompt.promptForChoice("Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes"): return utils.deleteFile(localArchiveAbsPath) and self.fetchBinary(downloadRetriesLeft=downloadRetriesLeft-1) if createingCache: raise BlueprintException(msg, self.package) return False self.subinfo.buildPrefix = latest.buildPrefix self.subinfo.isCachedBuild = True if not (self.cleanImage() and utils.unpackFile(localArchivePath, localArchiveName, self.imageDir()) and self.internalPostInstall() and self.postInstall() and self.qmerge() and self.internalPostQmerge() and self.postQmerge()): return False return True return False @staticmethod def getFileListFromDirectory(imagedir, filePaths): """ create a file list containing hashes """ ret = [] algorithm = CraftHash.HashAlgorithm.SHA256 for filePath in filePaths: relativeFilePath = os.path.relpath(filePath, imagedir) digest = algorithm.stringPrefix() + CraftHash.digestFile(filePath, algorithm) ret.append((relativeFilePath, digest)) return ret @staticmethod def unmergeFileList(rootdir, fileList): """ delete files in the fileList if has matches """ for filename, filehash in fileList: fullPath = os.path.join(rootdir, os.path.normcase(filename)) if os.path.isfile(fullPath) or os.path.islink(fullPath): if filehash: algorithm = CraftHash.HashAlgorithm.getAlgorithmFromPrefix(filehash) currentHash = algorithm.stringPrefix() + CraftHash.digestFile(fullPath, algorithm) if not filehash or currentHash == filehash: OsUtils.rm(fullPath, True) else: CraftCore.log.warning( f"We can't remove {fullPath} as its hash has changed," f" that usually implies that the file was modified or replaced") elif not os.path.isdir(fullPath) and os.path.lexists(fullPath): CraftCore.log.debug(f"Remove a dead symlink {fullPath}") OsUtils.rm(fullPath, True) elif not os.path.isdir(fullPath): CraftCore.log.warning("file %s does not exist" % fullPath) containingDir = os.path.dirname(fullPath) if os.path.exists(containingDir) and not os.listdir(containingDir): CraftCore.log.debug(f"Delete empty dir {containingDir}") utils.rmtree(containingDir) def runAction(self, command): functions = {"fetch": "fetch", "cleanimage": "cleanImage", "cleanbuild": "cleanBuild", "unpack": "unpack", "compile": "compile", "configure": "configure", "make": "make", "install": ["install", "internalPostInstall"], "post-install": "postInstall", "test": "unittest", "qmerge": ["qmerge", "internalPostQmerge"], "post-qmerge": "postQmerge", "unmerge": "unmerge", "package": "createPackage", "createpatch": "createPatch", "checkdigest": "checkDigest", "fetch-binary": "fetchBinary"} if command in functions: try: steps = functions[command] if not isinstance(steps, list): steps = [steps] for step in steps: if not getattr(self, step)(): return False except AttributeError as e: raise BlueprintException(str(e), self.package, e) - else: CraftCore.log.error("command %s not understood" % command) return False return True diff --git a/bin/Source/GitSource.py b/bin/Source/GitSource.py index 45c710b29..bbde436ab 100644 --- a/bin/Source/GitSource.py +++ b/bin/Source/GitSource.py @@ -1,228 +1,227 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # copyright (c) 2009 Ralf Habacker # copyright (c) 2009 Patrick Spendrin # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # git support import io from Source.VersionSystemSourceBase import * ## \todo requires installed git package -> add suport for installing packages class GitSource(VersionSystemSourceBase): """git support""" def __init__(self, subinfo=None): CraftCore.debug.trace('GitSource __init__') if subinfo: self.subinfo = subinfo VersionSystemSourceBase.__init__(self) def __getCurrentBranch(self): if os.path.exists(self.checkoutDir()): with io.StringIO() as tmp: self.__git("rev-parse", ["--abbrev-ref", "HEAD"], stdout=tmp) return tmp.getvalue().strip() return None def __isLocalBranch(self, branch): if os.path.exists(self.checkoutDir()): with io.StringIO() as tmp: self.__git("for-each-ref", ["--format=%(refname:short)", "refs/heads/*"], stdout=tmp) return branch in tmp.getvalue().strip().split("\n") return False def __isTag(self, _tag): if os.path.exists(self.checkoutDir()): with io.StringIO() as tmp: self.__git("tag", stdout=tmp) return _tag in tmp.getvalue().strip().split("\n") return False def __getCurrentRevision(self): """return the revision returned by git show""" # run the command branch = self.__getCurrentBranch() if not self.__isTag(branch): - # open a temporary file - do not use generic tmpfile because this doesn't give a good file object with python with io.StringIO() as tmp: self.__git("rev-parse", ["--short", "HEAD"], stdout=tmp) return f"{branch}-{tmp.getvalue().strip()}" else: # in case this is a tag, print out the tag version return branch def __git(self, command, args=None, logCommand=False, **kwargs): """executes a git command in a shell. Default for cwd is self.checkoutDir()""" parts = ["git", command] if command in ("clone", "checkout", "fetch", "pull", "submodule"): if CraftCore.debug.verbose() < 0: parts += ["-q"] else: kwargs["displayProgress"] = True else: kwargs["logCommand"] = logCommand if args: parts += args if not kwargs.get("cwd"): kwargs["cwd"] = self.checkoutDir() return utils.system(parts, **kwargs) def fetch(self): CraftCore.debug.trace('GitSource fetch') # get the path where the repositories should be stored to repopath = self.repositoryUrl() CraftCore.log.debug("fetching %s" % repopath) # in case you need to move from a read only Url to a writeable one, here it gets replaced repopath = repopath.replace("[git]", "") repoString = utils.replaceVCSUrl(repopath) [repoUrl, repoBranch, repoTag] = utils.splitVCSUrl(repoString) # override tag if self.subinfo.options.dynamic.revision: repoTag = self.subinfo.options.dynamic.revision repoBranch = "" # override the branch if self.subinfo.options.dynamic.branch: repoBranch = self.subinfo.options.dynamic.branch if repoTag and repoBranch: CraftCore.log.error(f"Your not allowed to specify a branch and a tag: branch -> {repoBranch}, tag -> {repoTag}") return False if not repoBranch and not repoTag: repoBranch = "master" # only run if wanted (e.g. no --offline is given on the commandline) if (self.noFetch): CraftCore.log.debug("skipping git fetch (--offline)") return True else: ret = True checkoutDir = self.checkoutDir() # if we only have the checkoutdir but no .git within, # clean this up first if os.path.exists(checkoutDir) \ and not os.path.exists(os.path.join(checkoutDir, ".git")): os.rmdir(checkoutDir) if os.path.isdir(checkoutDir): if not repoTag: ret = self.__git("fetch") \ and self.__git("checkout", [repoBranch or "master"]) \ and self.__git("merge") if self.subinfo.options.fetch.checkoutSubmodules: ret = ret and self.__git("submodule", ["update", "--init", "--recursive"]) else: args = [] # it doesn't exist so clone the repo os.makedirs(checkoutDir) # first try to replace with a repo url from etc/blueprints/crafthosts.conf if self.subinfo.options.fetch.checkoutSubmodules: args += ["--recursive"] ret = self.__git('clone', args + [repoUrl, self.checkoutDir()]) # if a branch is given, we should check first if the branch is already downloaded # locally, otherwise we can track the remote branch if ret and repoBranch and not repoTag: if not self.__isLocalBranch(repoBranch): track = ["--track", f"origin/{repoBranch}"] else: track = [repoBranch] ret = self.__git("checkout", track) # we can have tags or revisions in repoTag if ret and repoTag: if self.__isTag(repoTag): if not self.__isLocalBranch("_" + repoTag): ret = self.__git('checkout', ['-b', f"_{repoTag}", repoTag]) else: ret = self.__git('checkout', [f"_{repoTag}"]) else: # unknown tag, try to fetch it first self.__git('fetch', ['--tags']) ret = self.__git('checkout', [repoTag]) return ret def applyPatch(self, fileName, patchdepth, unusedSrcDir=None): """apply single patch o git repository""" CraftCore.debug.trace('GitSource ') if fileName: patchfile = os.path.join(self.packageDir(), fileName) return self.__git('apply', ['--ignore-space-change', '-p', str(patchdepth), patchfile], logCommand=True) return True def createPatch(self): """create patch file from git source into the related package dir. The patch file is named autocreated.patch""" CraftCore.debug.trace('GitSource createPatch') patchFileName = os.path.join(self.packageDir(), "%s-%s.patch" % \ (self.package.name, str(datetime.date.today()).replace('-', ''))) CraftCore.log.debug("git diff %s" % patchFileName) with open(patchFileName, 'wt+') as patchFile: return self.__git('diff', stdout=patchFile) def sourceVersion(self): """print the revision returned by git show""" CraftCore.debug.trace('GitSource sourceVersion') return self.__getCurrentRevision() def checkoutDir(self, index=0): CraftCore.debug.trace('GitSource checkoutDir') return VersionSystemSourceBase.checkoutDir(self, index) def sourceDir(self, index=0): CraftCore.debug.trace('GitSource sourceDir') repopath = self.repositoryUrl() # in case you need to move from a read only Url to a writeable one, here it gets replaced repopath = repopath.replace("[git]", "") sourcedir = self.checkoutDir(index) if self.subinfo.hasTargetSourcePath(): sourcedir = os.path.join(sourcedir, self.subinfo.targetSourcePath()) CraftCore.log.debug("using sourcedir: %s" % sourcedir) parent, child = os.path.split(sourcedir) return os.path.join(CraftShortPath(parent).shortPath, child) def getUrls(self): """print the url where to clone from and the branch/tag/hash""" # in case you need to move from a read only Url to a writeable one, here it gets replaced repopath = self.repositoryUrl().replace("[git]", "") repoString = utils.replaceVCSUrl(repopath) [repoUrl, repoBranch, repoTag] = utils.splitVCSUrl(repoString) if not repoBranch and not repoTag: repoBranch = "master" print('|'.join([repoUrl, repoBranch, repoTag])) return True