diff --git a/bin/CraftCommands.py b/bin/CraftCommands.py index da1ad950a..647906e7d 100644 --- a/bin/CraftCommands.py +++ b/bin/CraftCommands.py @@ -1,357 +1,368 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import subprocess import tempfile import CraftBase from Blueprints.CraftDependencyPackage import CraftDependencyPackage, DependencyType from Blueprints.CraftVersion import CraftVersion from Blueprints.CraftPackageObject import CraftPackageObject from Utils.CraftTitleUpdater import CraftTitleUpdater from Utils import CraftTimer from options import * import glob import utils def doExec(package, action): with CraftTimer.Timer("%s for %s" % (action, package), 1): CraftCore.debug.step("Action: %s for %s" % (action, package)) ret = package.instance.runAction(action) if not ret: if action == "fetch-binary": CraftCore.debug.step(f"{package} not found in cache") return False CraftCore.log.warning(f"Action: {action} for {package}:{package.version} FAILED") return ret def handlePackage(package, buildAction, directTargets): with CraftTimer.Timer(f"HandlePackage {package}", 3) as timer: success = True actions = [] timer.hook = lambda : utils.notify(f"Craft {buildAction} {'succeeded' if success else 'failed'}", f"{package} after {timer}", buildAction) CraftCore.debug.debug_line() CraftCore.debug.step(f"Handling package: {package}, action: {buildAction}") if buildAction == "all": if CraftCore.settings.getboolean("Packager", "UseCache", "False"): if doExec(package, "fetch-binary"): return True actions = ["fetch", "unpack", "compile", "cleanimage", "install", "post-install"] if CraftCore.settings.getboolean("ContinuousIntegration", "ClearBuildFolder", False): actions += ["cleanbuild"] actions += ["qmerge", "post-qmerge"] if CraftCore.settings.getboolean("Packager", "CreateCache"): onlyDirect = CraftCore.settings.getboolean("Packager", "CacheDirectTargetsOnly") if not onlyDirect or (onlyDirect and package in directTargets): actions += ["package"] else: actions = [buildAction] for action in actions: success = doExec(package, action) if not success: return False return True def resolvePackage(packageNames : [str], version : str=None) -> [CraftPackageObject]: package = CraftPackageObject(None) def resolveChildren(child): if child.isCategory(): for c in child.children.values(): resolveChildren(c) else: if version: UserOptions.addPackageOption(child, "version", version) package.children[child.name] = child for packageName in packageNames: child = CraftPackageObject.get(packageName) if not child: raise BlueprintNotFoundException(packageName) resolveChildren(child) return package def setOption(packageNames : [str], option : str) -> bool: if "=" not in option: CraftCore.log.error(f"Invalid option {option}") return False key, value = option.split("=", 1) for name in packageNames: package = CraftPackageObject.get(name) if not package: raise BlueprintNotFoundException(name) # create instance to make sure options are registered if not package.isCategory(): package.instance if not package: raise BlueprintNotFoundException(name) options = UserOptions.get(package) if not options.setOption(key, value): return False CraftCore.log.info(f"[{package}]\n{key}={getattr(options, key)}") return True def addBlueprintsRepository(url : str, args) -> bool: templateDir = os.path.join(CraftCore.standardDirs.craftBin(), "..", "internal_blueprints" ) with tempfile.TemporaryDirectory() as tmp: iniPath = os.path.join(tmp, "version.ini") parser = configparser.ConfigParser() parser.read(iniPath) parser.add_section("General") parser["General"]["branches"] = "master" parser["General"]["defaulttarget"] = "master" parser["General"]["gitUrl"] = url with open(iniPath, "wt+") as out: parser.write(out) CraftCore.settings.set("Blueprints", "Locations", templateDir) CraftCore.settings.set("InternalTemp", "add-bluprints-template.ini", iniPath) package = resolvePackage(["add-bluprints-template"]) return run(package, "fetch", args) def destroyCraftRoot() -> bool: settingsFiles = {"kdesettings.ini", "CraftSettings.ini", "BlueprintSettings.ini"} dirsToKeep = [CraftCore.standardDirs.downloadDir(), os.path.join(CraftCore.standardDirs.craftBin(), ".."), os.path.join(CraftCore.standardDirs.craftRoot(), "python"), CraftCore.standardDirs.blueprintRoot()] # dirs with possible interesting sub dirs maybeKeepDir = [ CraftCore.standardDirs.craftRoot(), CraftCore.standardDirs.etcDir(), os.path.join(CraftCore.standardDirs.etcDir(), "blueprints")# might contain blueprintRoot ] def deleteEntry(path): if utils.OsUtils.isLink(path): CraftCore.log.debug(f"Skipping symlink {path}") return if os.path.isdir(path): if any(os.path.exists(x) and os.path.samefile(path, x) for x in maybeKeepDir): CraftCore.log.debug(f"Path {path} in maybeKeepDir") for entry in os.listdir(path): deleteEntry(os.path.join(path, entry)) elif any(os.path.exists(x) and os.path.samefile(path, x) for x in dirsToKeep): CraftCore.log.debug(f"Path {path} in dirsToKeep") else: utils.cleanDirectory(path) utils.OsUtils.rmDir(path, True) else: if os.path.basename(path) not in settingsFiles: utils.OsUtils.rm(path, True) del CraftCore.installdb deleteEntry(CraftCore.standardDirs.craftRoot()) return True def readListFile(listFile): packageNames = [] parser = configparser.ConfigParser(allow_no_value=True) parser.read(listFile) for sections in parser.keys(): for packageName in parser[sections]: version = parser.get(sections, packageName) if version: UserOptions.setOptions([f"{packageName}.version={version}"]) packageNames.append(packageName) return packageNames def packageIsOutdated(package): installed = CraftCore.installdb.getInstalledPackages(package) if not installed: return True for pack in installed: version = pack.getVersion() if not version: continue cacheVersion = pack.getCacheVersion() if cacheVersion and cacheVersion != CraftBase.CraftBase.cacheVersion(): # can only happen for packages installed from cache return True return package.version != version def invoke(command : str, directTargets : [CraftPackageObject]) -> bool: args = {} key = command argsPattern = re.compile(r"(.+)\((.*)\)") argsMatch = argsPattern.findall(command) if argsMatch: key = argsMatch[0][0] args = eval(f"dict({','.join(argsMatch[0][1:])})") subs = key.split(".") for p in directTargets: instance = p.instance path = [] for sub in subs: path += [sub] if hasattr(instance, sub): attr = getattr(instance, sub) if callable(attr): instance = attr(**args) else: instance = attr else: CraftCore.debug.printOut(f"{p} has no member {'.'.join(path)}", file=sys.stderr) return False CraftCore.log.debug(f"--get {command} on {p} -> {type(instance)}:{instance}") CraftCore.debug.printOut(instance) return True def run(package : [CraftPackageObject], action : str, args) -> bool: if package.isIgnored(): CraftCore.log.info(f"Skipping package because it has been ignored: {package}") return True directTargets = package.children.values() CraftCore.state.directTargets = directTargets if action == "get": return invoke(args.get, directTargets) - if action == "install-to-desktop": + elif action == "install-to-desktop": return installToDektop(directTargets) + elif action == "print-files": + return printFiles(directTargets) elif args.resolve_deps or action in ["all", "install-deps"]: # work on the dependencies depPackage = CraftDependencyPackage(package) if args.resolve_deps: if not args.resolve_deps.capitalize() in DependencyType.__members__: CraftCore.log.error(f"Invalid dependency type {args.resolve_deps}, valid types are {DependencyType.__members__}") return False depType = DependencyType.__getattr__(args.resolve_deps.capitalize()) - print(depType) elif action == "install-deps": depType = DependencyType.Both else: depType = DependencyType.All depList = depPackage.getDependencies(depType=depType) packages = [] if not args.resolve_deps: for item in depList: if (args.ignoreInstalled and item in directTargets) or packageIsOutdated(item): packages.append(item) CraftCore.log.debug(f"dependency: {item}") elif item in directTargets: CraftCore.debug.step(f"{item} is up to date, nothing to do") else: packages = depList if not packages: CraftCore.log.debug("") if action == "install-deps": # we don't intend to build the package itself for x in directTargets: packages.remove(x) CraftTitleUpdater.usePackageProgressTitle(packages) while packages: info = packages[0] # in case we only want to see which packages are still to be build, simply return the package name if args.probe: CraftCore.log.warning(f"pretending {info}: {info.version}") else: if CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): CraftCore.debug.debug_line() CraftCore.log.info(f"Status: {CraftTitleUpdater.instance}") if action in ["install-deps"]: action = "all" if not handlePackage(info, action, directTargets=directTargets): CraftCore.log.error(f"fatal error: package {info} {action} failed") return False packages.pop(0) else: for info in directTargets: if not handlePackage(info, action, directTargets=directTargets): return False CraftCore.debug.new_line() return True def cleanBuildFiles(cleanArchives, cleanImages, cleanInstalledImages, cleanBuildDir, packages): def cleanDir(dir): CraftCore.debug.printOut(f"Cleaning: {dir}") utils.cleanDirectory(dir) os.rmdir(dir) for p in packages: package = CraftPackageObject.get(p.path) if not package or package.isCategory(): continue CraftCore.log.debug(f"Checking package for unused files: {p.path}") instance = package.instance version = instance.version if version: imageGlob = instance.imageDir().replace(version, "*") builddirGlob = instance.buildDir().replace(version, "*") else: imageGlob = instance.imageDir() builddirGlob = instance.buildDir() # image directories if cleanImages: for dir in glob.glob(imageGlob): if package.isInstalled and not cleanInstalledImages: if dir == instance.imageDir(): continue cleanDir(dir) # archive directory if cleanArchives and os.path.exists(instance.archiveDir()): cleanDir(instance.archiveDir()) # build directory if cleanBuildDir: for dir in glob.glob(builddirGlob): cleanDir(dir) def updateInstalled(args) -> bool: ENV_KEY = "CRAFT_CORE_UPDATED" if ENV_KEY not in os.environ: os.environ[ENV_KEY] = "1" # update the core if not run(CraftPackageObject.get("craft"), "all", args): return False # close the log file and the db CraftTitleUpdater.instance.stop() CraftCore.debug._fileHandler.close() del CraftCore.installdb return subprocess.call([sys.executable] + sys.argv) == 0 else: package = CraftPackageObject(None) for packageName, _ in CraftCore.installdb.getDistinctInstalled(): p = CraftPackageObject.get(packageName) if p: package.children[p.name] = p return run(package, "all", args) def installToDektop(packages): CraftCore.settings.set("Packager", "PackageType", "DesktopEntry") for p in packages: if not p.instance.createPackage(): return False return True + +def printFiles(packages): + for p in packages: + packageList = CraftCore.installdb.getInstalledPackages(p) + for package in packageList: + fileList = package.getFiles() + fileList.sort() + for file in fileList: + CraftCore.log.info(file[0]) + return True diff --git a/bin/Package/PackageBase.py b/bin/Package/PackageBase.py index 5b92faebb..8ac37d86a 100644 --- a/bin/Package/PackageBase.py +++ b/bin/Package/PackageBase.py @@ -1,225 +1,215 @@ # # copyright (c) 2009 Ralf Habacker # from pathlib import Path from CraftBase import * from CraftCompiler import * from InstallDB import * from Blueprints.CraftPackageObject import * from Utils import CraftHash, GetFiles, CraftChoicePrompt from Utils.CraftManifest import CraftManifest import json class PackageBase(CraftBase): """ provides a generic interface for packages and implements the basic stuff for all packages """ # uses the following instance variables # todo: place in related ...Base # rootdir -> CraftBase # package -> PackageBase # force -> PackageBase # category -> PackageBase # version -> PackageBase # packagedir -> PackageBase # imagedir -> PackageBase def __init__(self): CraftCore.log.debug("PackageBase.__init__ called") CraftBase.__init__(self) def qmerge(self): """mergeing the imagedirectory into the filesystem""" ## \todo is this the optimal place for creating the post install scripts ? if self.package.isInstalled: self.unmerge() copiedFiles = [] # will be populated by the next call if not utils.copyDir(self.imageDir(), CraftCore.standardDirs.craftRoot(), copiedFiles=copiedFiles): return False # add package to installed database -> is this not the task of the manifest files ? revision = self.sourceRevision() package = CraftCore.installdb.addInstalled(self.package, self.version, revision=revision) fileList = self.getFileListFromDirectory(CraftCore.standardDirs.craftRoot(), copiedFiles) package.addFiles(fileList) package.install() if (CraftCore.settings.getboolean("Packager", "CreateCache") or CraftCore.settings.getboolean("Packager", "UseCache")): package.setCacheVersion(self.cacheVersion()) return True def unmerge(self): """unmergeing the files from the filesystem""" CraftCore.log.debug("Packagebase unmerge called") packageList = CraftCore.installdb.getInstalledPackages(self.package) for package in packageList: fileList = package.getFilesWithHashes() self.unmergeFileList(CraftCore.standardDirs.craftRoot(), fileList) package.uninstall() return True def createImportLibs(self, pkgName): """create the import libraries for the other compiler(if ANSI-C libs)""" basepath = os.path.join(self.installDir()) utils.createImportLibs(pkgName, basepath) - def printFiles(self): - packageList = CraftCore.installdb.getInstalledPackages(self.package) - for package in packageList: - fileList = package.getFiles() - fileList.sort() - for file in fileList: - print(file[0]) - return True - def fetchBinary(self, downloadRetriesLeft=3) -> bool: if self.subinfo.options.package.disableBinaryCache: return False for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.") if url == self.cacheLocation(): fileUrl = f"{url}/manifest.json" if os.path.exists(fileUrl): with open(fileUrl, "rt", encoding="UTF-8") as f: manifest = CraftManifest.fromJson(json.load(f)) else: continue else: manifest = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) fileEntry = manifest.get(str(self)).files files = [] for f in fileEntry: if f.version == self.version: files.append(f) if not files: CraftCore.log.debug(f"Could not find {self}={self.version} in {url}") continue latest = files[0] if latest.configHash and latest.configHash != self.subinfo.options.dynamic.configHash(): CraftCore.log.warning("Failed to restore package, configuration missmatch") return False if url != self.cacheLocation(): downloadFolder = self.cacheLocation(os.path.join(CraftCore.standardDirs.downloadDir(), "cache")) else: downloadFolder = self.cacheLocation() localArchiveAbsPath = OsUtils.toNativePath(os.path.join(downloadFolder, latest.fileName)) localArchivePath, localArchiveName = os.path.split(localArchiveAbsPath) if url != self.cacheLocation(): if not os.path.exists(localArchiveAbsPath): os.makedirs(localArchivePath, exist_ok=True) fUrl = f"{url}/{latest.fileName}" if not GetFiles.getFile(fUrl, localArchivePath, localArchiveName): CraftCore.log.warning(f"Failed to fetch {fUrl}") return False elif not os.path.isfile(localArchiveAbsPath): continue if not CraftHash.checkFilesDigests(localArchivePath, [localArchiveName], digests=latest.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): CraftCore.log.warning(f"Hash did not match, {localArchiveName} might be corrupted") if downloadRetriesLeft and CraftChoicePrompt.promptForChoice("Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes"): return utils.deleteFile(localArchiveAbsPath) and self.fetchBinary(downloadRetriesLeft=downloadRetriesLeft-1) return False self.subinfo.buildPrefix = latest.buildPrefix self.subinfo.isCachedBuild = True if not (self.cleanImage() and utils.unpackFile(localArchivePath, localArchiveName, self.imageDir()) and self.internalPostInstall() and self.postInstall() and self.qmerge() and self.internalPostQmerge() and self.postQmerge()): return False return True return False @staticmethod def getFileListFromDirectory(imagedir, filePaths): """ create a file list containing hashes """ ret = [] algorithm = CraftHash.HashAlgorithm.SHA256 for filePath in filePaths: relativeFilePath = os.path.relpath(filePath, imagedir) digest = algorithm.stringPrefix() + CraftHash.digestFile(filePath, algorithm) ret.append((relativeFilePath, digest)) return ret @staticmethod def unmergeFileList(rootdir, fileList): """ delete files in the fileList if has matches """ for filename, filehash in fileList: fullPath = os.path.join(rootdir, os.path.normcase(filename)) if os.path.isfile(fullPath) or os.path.islink(fullPath): if filehash: algorithm = CraftHash.HashAlgorithm.getAlgorithmFromPrefix(filehash) currentHash = algorithm.stringPrefix() + CraftHash.digestFile(fullPath, algorithm) if not filehash or currentHash == filehash: OsUtils.rm(fullPath, True) else: CraftCore.log.warning( f"We can't remove {fullPath} as its hash has changed," f" that usually implies that the file was modified or replaced") elif not os.path.isdir(fullPath) and os.path.lexists(fullPath): CraftCore.log.debug(f"Remove a dead symlink {fullPath}") OsUtils.rm(fullPath, True) elif not os.path.isdir(fullPath): CraftCore.log.warning("file %s does not exist" % fullPath) containingDir = os.path.dirname(fullPath) if os.path.exists(containingDir) and not os.listdir(containingDir): CraftCore.log.debug(f"Delete empty dir {containingDir}") utils.rmtree(containingDir) def runAction(self, command): functions = {"fetch": "fetch", "cleanimage": "cleanImage", "cleanbuild": "cleanBuild", "unpack": "unpack", "compile": "compile", "configure": "configure", "make": "make", "install": ["install", "internalPostInstall"], "post-install": "postInstall", "test": "unittest", "qmerge": ["qmerge", "internalPostQmerge"], "post-qmerge": "postQmerge", "unmerge": "unmerge", "package": "createPackage", "createpatch": "createPatch", - "print-files": "printFiles", "checkdigest": "checkDigest", "fetch-binary": "fetchBinary"} if command in functions: try: steps = functions[command] if not isinstance(steps, list): steps = [steps] for step in steps: if not getattr(self, step)(): return False except AttributeError as e: raise BlueprintException(str(e), self.package, e) else: CraftCore.log.error("command %s not understood" % command) return False return True