diff --git a/bin/BuildSystem/BuildSystemBase.py b/bin/BuildSystem/BuildSystemBase.py index c3151aa42..a2ff8f37b 100644 --- a/bin/BuildSystem/BuildSystemBase.py +++ b/bin/BuildSystem/BuildSystemBase.py @@ -1,304 +1,304 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # Copyright 2009 Ralf Habacker # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. """ \package BuildSystemBase""" import glob import io import multiprocessing import os import re import subprocess from pathlib import Path from CraftBase import * from CraftOS.osutils import OsUtils class BuildSystemBase(CraftBase): """provides a generic interface for build systems and implements all stuff for all build systems""" PatchableFile = {".service", ".pc", ".pri", ".prl", ".cmake", ".conf", ".sh", ".bat", ".cmd", ".ini", ".pl", ".pm", ".la", ".py"} def __init__(self, typeName=""): """constructor""" CraftBase.__init__(self) self.supportsNinja = False self.supportsCCACHE = CraftCore.settings.getboolean("Compile", "UseCCache", False) and CraftCore.compiler.isGCCLike() self.supportsClang = True self.buildSystemType = typeName @property def makeProgram(self) -> str: if self.subinfo.options.make.supportsMultijob: if self.supportsNinja and CraftCore.settings.getboolean("Compile", "UseNinja", False) and CraftCore.cache.findApplication("ninja"): return "ninja" if ("Compile", "MakeProgram") in CraftCore.settings: makeProgram = CraftCore.settings.get("Compile", "MakeProgram") CraftCore.log.debug(f"set custom make program: {makeProgram}") if CraftCore.cache.findApplication(makeProgram): return makeProgram else: CraftCore.log.warning(f"Failed to find {CraftCore.settings.get('Compile', 'MakeProgram')}") elif not self.subinfo.options.make.supportsMultijob: if "MAKE" in os.environ: del os.environ["MAKE"] if OsUtils.isWin(): if CraftCore.compiler.isMSVC() or CraftCore.compiler.isIntel(): if self.subinfo.options.make.supportsMultijob: return "jom" else: return "nmake" elif CraftCore.compiler.isMinGW(): return "mingw32-make" else: CraftCore.log.critical(f"unknown {CraftCore.compiler} compiler") elif OsUtils.isUnix(): return "make" def compile(self): """convencience method - runs configure() and make()""" configure = getattr(self, 'configure') make = getattr(self, 'make') return configure() and make() def configureSourceDir(self): """returns source dir used for configure step""" # pylint: disable=E1101 # this class never defines self.source, that happens only # in MultiSource. sourcedir = self.sourceDir() if self.subinfo.hasConfigurePath(): sourcedir = os.path.join(sourcedir, self.subinfo.configurePath()) return sourcedir def configureOptions(self, defines=""): """return options for configure command line""" if self.subinfo.options.configure.args != None: defines += " %s" % self.subinfo.options.configure.args if self.supportsCCACHE: defines += " %s" % self.ccacheOptions() if CraftCore.compiler.isClang() and self.supportsClang: defines += " %s" % self.clangOptions() return defines def makeOptions(self, args): """return options for make command line""" defines = [] if self.subinfo.options.make.ignoreErrors: defines.append("-i") makeProgram = self.makeProgram if makeProgram == "ninja": if CraftCore.debug.verbose() > 0: defines.append("-v") else: if CraftCore.debug.verbose() > 0: defines += ["VERBOSE=1", "V=1"] if self.subinfo.options.make.supportsMultijob and makeProgram != "nmake" : if makeProgram not in {"ninja", "jom"} or ("Compile", "Jobs") in CraftCore.settings: defines += ["-j", str(CraftCore.settings.get("Compile", "Jobs", multiprocessing.cpu_count()))] if args: defines.append(args) return " ".join(defines) def configure(self): return True def make(self): return True def install(self) -> bool: return self.cleanImage() def unittest(self): """running unittests""" return True def ccacheOptions(self): return "" def clangOptions(self): return "" def _fixInstallPrefix(self, prefix=CraftStandardDirs.craftRoot()): CraftCore.log.debug(f"Begin: fixInstallPrefix {self}: {prefix}") def stripPath(path): rootPath = os.path.splitdrive(path)[1] if rootPath.startswith(os.path.sep) or rootPath.startswith("/"): rootPath = rootPath[1:] return rootPath badPrefix = os.path.join(self.installDir(), stripPath(prefix)) if os.path.exists(badPrefix) and not os.path.samefile(self.installDir(), badPrefix): if not utils.mergeTree(badPrefix, self.installDir()): return False if CraftCore.settings.getboolean("QtSDK", "Enabled", False): qtDir = os.path.join(CraftCore.settings.get("QtSDK", "Path"), CraftCore.settings.get("QtSDK", "Version"), CraftCore.settings.get("QtSDK", "Compiler")) path = os.path.join(self.installDir(), stripPath(qtDir)) if os.path.exists(path) and not os.path.samefile(self.installDir(), path): if not utils.mergeTree(path, self.installDir()): return False if stripPath(prefix): oldPrefix = OsUtils.toUnixPath(stripPath(prefix)).split("/", 1)[0] utils.rmtree(os.path.join(self.installDir(), oldPrefix)) CraftCore.log.debug(f"End: fixInstallPrefix {self}") return True def patchInstallPrefix(self, files: [str], oldPaths: [Path] = None, newPath: Path = Path(CraftCore.standardDirs.craftRoot())) -> bool: if not isinstance(oldPaths, list): oldPaths = [oldPaths] elif not oldPaths: oldPaths = [self.subinfo.buildPrefix] oldPaths = [Path(x).as_posix() for x in oldPaths] newValue = Path(newPath).as_posix().encode() for fileName in files: if not os.path.exists(fileName): CraftCore.log.warning(f"File {fileName} not found.") return False with open(fileName, "rb") as f: content = f.read() dirty = False for oldPath in oldPaths: assert os.path.isabs(oldPath) # allow front and back slashes oldPathPat = oldPath.replace("/", r"[/\\]+") # capture firs seperator oldPathPat = oldPathPat.replace(r"[/\\]+", r"(/+|\\+)", 1) oldPathPat = f"({oldPathPat})" oldPathPat = re.compile(oldPathPat.encode()) for match in set(oldPathPat.findall(content)): dirty = True oldPath = match[0] newPath = newValue.replace(b"/", match[1]) if oldPath != newPath: CraftCore.log.info(f"Patching {fileName}: replacing {oldPath} with {newPath}") content = content.replace(oldPath, newPath) else: CraftCore.log.debug(f"Skip Patching {fileName}: prefix is unchanged {newPath}") if dirty: - with utils.makeWritable(fileName): + with utils.makeTemporaryWritable(fileName): with open(fileName, "wb") as f: f.write(content) return True def internalPostInstall(self): if not super().internalPostInstall(): return False # fix absolute symlinks for sym in utils.filterDirectoryContent(self.installDir(), lambda x, root: x.is_symlink(), lambda x, root: True, allowBadSymlinks=True): target = Path(os.readlink(sym)) if target.is_absolute(): sym = Path(sym) target = Path(self.imageDir()) / target.relative_to(CraftCore.standardDirs.craftRoot()) sym.unlink() # we can't use relative_to here sym.symlink_to(os.path.relpath(target, sym.parent)) # a post install routine to fix the prefix (make things relocatable) newPrefix = OsUtils.toUnixPath(CraftCore.standardDirs.craftRoot()) oldPrefixes = [self.subinfo.buildPrefix] if CraftCore.compiler.isWindows: oldPrefixes += [OsUtils.toMSysPath(self.subinfo.buildPrefix)] files = utils.filterDirectoryContent(self.installDir(), whitelist=lambda x, root: Path(x).suffix in BuildSystemBase.PatchableFile, blacklist=lambda x, root: True) if not self.patchInstallPrefix(files, oldPrefixes, newPrefix): return False binaryFiles = list(utils.filterDirectoryContent(self.installDir(), lambda x, root: utils.isBinary(x.path), lambda x, root: True)) if (CraftCore.compiler.isMacOS and os.path.isdir(self.installDir())): for f in binaryFiles: if os.path.islink(f): continue # replace the old prefix or add it if missing if not utils.system(["install_name_tool", "-rpath", os.path.join(self.subinfo.buildPrefix, "lib"), os.path.join(newPrefix, "lib"), f], logCommand=False): utils.system(["install_name_tool", "-add_rpath", os.path.join(newPrefix, "lib"), f], logCommand=False) # update prefix if self.subinfo.buildPrefix != newPrefix: if os.path.splitext(f)[1] in {".dylib", ".so"}: # fix dylib id with io.StringIO() as log: utils.system(["otool", "-D", f], stdout=log, logCommand=False) oldId = log.getvalue().strip().split("\n") # the first line is the file name # the second the id, if we only get one line, there is no id to fix if len(oldId) == 2: oldId = oldId[1].strip() newId = oldId.replace(self.subinfo.buildPrefix, newPrefix) if newId != oldId: if not utils.system(["install_name_tool", "-id", newId, f], logCommand=False): return False # fix dependencies for dep in utils.getLibraryDeps(f): if dep.startswith(self.subinfo.buildPrefix): newDep = dep.replace(self.subinfo.buildPrefix, newPrefix) if newDep != dep: if not utils.system(["install_name_tool", "-change", dep, newDep, f], logCommand=False): return False # Install pdb files on MSVC if they are not found next to the dll # skip if we are a release build or from cache if not self.subinfo.isCachedBuild: if self.buildType() in {"RelWithDebInfo", "Debug"}: if CraftCore.compiler.isMSVC(): for f in binaryFiles: if not os.path.exists(f"{os.path.splitext(f)[0]}.pdb"): pdb = utils.getPDBForBinary(f) if not pdb: CraftCore.log.warning(f"Could not find a PDB for {f}") continue if not os.path.exists(pdb): CraftCore.log.warning(f"PDB {pdb} for {f} does not exist") continue pdbDestination = os.path.join(os.path.dirname(f), os.path.basename(pdb)) CraftCore.log.info(f"Install pdb: {pdbDestination} for {os.path.basename(f)}") utils.copyFile(pdb, pdbDestination, linkOnly=False) else: if not self.subinfo.options.package.disableStriping: for f in binaryFiles: utils.strip(f) # sign the binaries if we can if CraftCore.compiler.isWindows and CraftCore.settings.getboolean("CodeSigning", "SignCache", False): if not utils.sign(binaryFiles): return False return True diff --git a/bin/Packager/MacBasePackager.py b/bin/Packager/MacBasePackager.py index fef1531db..f5668330a 100644 --- a/bin/Packager/MacBasePackager.py +++ b/bin/Packager/MacBasePackager.py @@ -1,397 +1,397 @@ from Packager.CollectionPackagerBase import * from Blueprints.CraftPackageObject import CraftPackageObject from Utils import CraftHash from pathlib import Path import contextlib import io import subprocess import stat import glob class MacBasePackager( CollectionPackagerBase ): @InitGuard.init_once def __init__(self, whitelists, blacklists): CollectionPackagerBase.__init__(self, whitelists, blacklists) def internalCreatePackage(self, defines, seperateSymbolFiles=False, packageSymbols=False): """ create a package """ CraftCore.log.debug("packaging using the MacDMGPackager") # TODO: provide an image with dbg files if not super().internalCreatePackage(defines, seperateSymbolFiles=seperateSymbolFiles, packageSymbols=packageSymbols): return False appPath = self.getMacAppPath(defines) archive = Path(self.archiveDir()) CraftCore.log.info(f"Packaging {appPath}") CraftCore.log.info("Clean up frameworks") for framework in utils.filterDirectoryContent(archive / "lib", handleAppBundleAsFile=True, whitelist=lambda x, root: x.name.endswith(".framework"), blacklist=lambda x, root: True): rubbish = [] framework = Path(framework) rubbish += glob.glob(str(framework / "*.prl")) rubbish += glob.glob(str(framework / "Headers")) for r in rubbish: r = Path(r) if r.is_symlink(): if framework not in r.parents: raise Exception(f"Evil symlink detected: {r}") utils.deleteFile(r) r = r.resolve() if r.is_dir(): utils.rmtree(r) else: utils.deleteFile(r) targetLibdir = os.path.join(appPath, "Contents", "Frameworks") utils.createDir(targetLibdir) moveTargets = [ (archive / "lib/plugins", appPath / "Contents/PlugIns"), (archive / "plugins", appPath / "Contents/PlugIns"), (archive / "share", appPath / "Contents/Resources"), (archive / "translations", appPath / "Contents/Resources/Translations"), (archive / "bin", appPath / "Contents/MacOS"), (archive / "libexec", appPath / "Contents/MacOS"), (archive / "lib/libexec/kf5", appPath / "Contents/MacOS"), (archive / "lib/libexec", appPath / "Contents/MacOS"), (archive / "lib", targetLibdir), ] if archive not in appPath.parents: moveTargets += [(os.path.join(archive, "bin"), os.path.join(appPath, "Contents", "MacOS"))] for src, dest in moveTargets: if os.path.exists(src): if not utils.mergeTree(src, dest): return False dylibbundler = MacDylibBundler(appPath) with utils.ScopedEnv({'DYLD_FALLBACK_LIBRARY_PATH': targetLibdir + ":" + os.path.join(CraftStandardDirs.craftRoot(), "lib")}): CraftCore.log.info("Bundling main binary dependencies...") mainBinary = Path(appPath, "Contents", "MacOS", defines['appname']) if not dylibbundler.bundleLibraryDependencies(mainBinary): return False binaries = list(utils.filterDirectoryContent(os.path.join(appPath, "Contents", "MacOS"), whitelist=lambda x, root: utils.isBinary(x.path) and x.name != defines["appname"], blacklist=lambda x, root: True)) for binary in binaries: CraftCore.log.info(f"Bundling dependencies for {binary}...") binaryPath = Path(binary) if not dylibbundler.bundleLibraryDependencies(binaryPath): return False # Fix up the library dependencies of files in Contents/Frameworks/ CraftCore.log.info("Bundling library dependencies...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/Frameworks"): return False CraftCore.log.info("Bundling plugin dependencies...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/PlugIns"): return False macdeployqt_multiple_executables_command = ["macdeployqt", appPath, "-always-overwrite", "-verbose=1"] for binary in binaries: macdeployqt_multiple_executables_command.append(f"-executable={binary}") if "qmldirs" in self.defines.keys() and isinstance(self.defines["qmldirs"], list): for qmldir in self.defines["qmldirs"]: macdeployqt_multiple_executables_command.append(f"-qmldir={qmldir}") if not utils.system(macdeployqt_multiple_executables_command): return False # macdeployqt might just have added some explicitly blacklisted files blackList = Path(self.packageDir(), "mac_blacklist.txt") if blackList.exists(): pattern = [self.read_blacklist(str(blackList))] # use it as whitelist as we want only matches, ignore all others matches = utils.filterDirectoryContent(appPath, whitelist=lambda x, root: utils.regexFileFilter(x, root, pattern), blacklist=lambda x, root:True) for f in matches: CraftCore.log.info(f"Remove blacklisted file: {f}") utils.deleteFile(f) # macdeployqt adds some more plugins so we fix the plugins after calling macdeployqt dylibbundler.checkedLibs = set() # ensure we check all libs again (but # we should not need to make any changes) CraftCore.log.info("Fixing plugin dependencies after macdeployqt...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/PlugIns"): return False CraftCore.log.info("Fixing library dependencies after macdeployqt...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/Frameworks"): return False # Finally sanity check that we don't depend on absolute paths from the builder CraftCore.log.info("Checking for absolute library paths in package...") found_bad_dylib = False # Don't exit immeditately so that we log all the bad libraries before failing: if not dylibbundler.areLibraryDepsOkay(mainBinary): found_bad_dylib = True CraftCore.log.error("Found bad library dependency in main binary %s", mainBinary) for binary in binaries: binaryPath = Path(binary) if not dylibbundler.areLibraryDepsOkay(binaryPath): found_bad_dylib = True CraftCore.log.error("Found bad library dependency in binary %s", binaryPath) if not dylibbundler.checkLibraryDepsRecursively("Contents/Frameworks"): CraftCore.log.error("Found bad library dependency in bundled libraries") found_bad_dylib = True if not dylibbundler.checkLibraryDepsRecursively("Contents/PlugIns"): CraftCore.log.error("Found bad library dependency in bundled plugins") found_bad_dylib = True if found_bad_dylib: CraftCore.log.error("Cannot not create .dmg since the .app contains a bad library depenency!") return False return utils.signMacApp(appPath) class MacDylibBundler(object): """ Bundle all .dylib files that are not provided by the system with the .app """ def __init__(self, appPath: str): # Avoid processing the same file more than once self.checkedLibs = set() self.appPath = appPath def _addLibToAppImage(self, libPath: Path) -> bool: assert libPath.is_absolute(), libPath libBasename = libPath.name targetPath = Path(self.appPath, "Contents/Frameworks/", libBasename) if targetPath.exists() and targetPath in self.checkedLibs: return True # Handle symlinks (such as libgit2.27.dylib -> libgit2.0.27.4.dylib): if libPath.is_symlink(): linkTarget = os.readlink(str(libPath)) CraftCore.log.info("Library dependency %s is a symlink to '%s'", libPath, linkTarget) if os.path.isabs(linkTarget): CraftCore.log.error("%s: Cannot handle absolute symlinks: '%s'", libPath, linkTarget) return False if ".." in linkTarget: CraftCore.log.error("%s: Cannot handle symlinks containing '..': '%s'", libPath, linkTarget) return False if libPath.resolve().parent != libPath.parent.resolve(): CraftCore.log.error("%s: Cannot handle symlinks to other directories: '%s' (%s vs %s)", libPath, linkTarget, libPath.resolve().parent, libPath.parent.resolve()) return False # copy the symlink and add the real file: utils.copyFile(str(libPath), str(targetPath), linkOnly=False) CraftCore.log.info("Added symlink '%s' (%s) to bundle -> %s", libPath, os.readlink(str(targetPath)), targetPath) self.checkedLibs.add(targetPath) symlinkTarget = libPath.with_name(os.path.basename(linkTarget)) CraftCore.log.info("Processing symlink target '%s'", symlinkTarget) if not self._addLibToAppImage(symlinkTarget): self.checkedLibs.remove(targetPath) return False # If the symlink target was processed, the symlink itself is also fine return True if not libPath.exists(): CraftCore.log.error("Library dependency '%s' does not exist", libPath) return False CraftCore.log.debug("Handling library dependency '%s'", libPath) if not targetPath.exists(): utils.copyFile(str(libPath), str(targetPath), linkOnly=False) CraftCore.log.info("Added library dependency '%s' to bundle -> %s", libPath, targetPath) if not self._fixupLibraryId(targetPath): return False for path in utils.getLibraryDeps(str(targetPath)): # check there aren't any references to the original location: if path == str(libPath): CraftCore.log.error("%s: failed to fix reference to original location for '%s'", targetPath, path) return False if not self.bundleLibraryDependencies(targetPath): CraftCore.log.error("%s: UNKNOWN ERROR adding '%s' into bundle", targetPath, libPath) return False if not os.path.exists(targetPath): CraftCore.log.error("%s: Library dependency '%s' doesn't exist after copying... Symlink error?", targetPath, libPath) return False self.checkedLibs.add(targetPath) return True @staticmethod def _updateLibraryReference(fileToFix: Path, oldRef: str, newRef: str = None) -> bool: if newRef is None: newRef = "@executable_path/../Frameworks/" + os.path.basename(oldRef) - with utils.makeWritable(fileToFix): + with utils.makeTemporaryWritable(fileToFix): if not utils.system(["install_name_tool", "-change", oldRef, newRef, str(fileToFix)], logCommand=False): CraftCore.log.error("%s: failed to update library dependency path from '%s' to '%s'", fileToFix, oldRef, newRef) return False return True @staticmethod def _getLibraryNameId(fileToFix: Path) -> str: libraryIdOutput = io.StringIO( subprocess.check_output(["otool", "-D", str(fileToFix)]).decode("utf-8").strip()) lines = libraryIdOutput.readlines() if len(lines) == 1: return "" # Should have exactly one line with the id now assert len(lines) == 2, lines return lines[1].strip() @classmethod def _fixupLibraryId(cls, fileToFix: Path): libraryId = cls._getLibraryNameId(fileToFix) if libraryId and os.path.isabs(libraryId): CraftCore.log.debug("Fixing library id name for %s", libraryId) - with utils.makeWritable(fileToFix): + with utils.makeTemporaryWritable(fileToFix): if not utils.system(["install_name_tool", "-id", os.path.basename(libraryId), str(fileToFix)], logCommand=False): CraftCore.log.error("%s: failed to fix absolute library id name for", fileToFix) return False return True def bundleLibraryDependencies(self, fileToFix: Path) -> bool: assert not fileToFix.is_symlink(), fileToFix if fileToFix.stat().st_nlink > 1: CraftCore.log.error("More than one hard link to library %s found! " "This might modify another accidentally.", fileToFix) CraftCore.log.info("Fixing library dependencies for %s", fileToFix) if not self._fixupLibraryId(fileToFix): return False # Ensure we have the current library ID since we need to skip it in the otool -L output libraryId = self._getLibraryNameId(fileToFix) for path in utils.getLibraryDeps(str(fileToFix)): if path == libraryId: # The first line of the otool output is (usually?) the library itself: # $ otool -L PlugIns/printsupport/libcocoaprintersupport.dylib: # PlugIns/printsupport/libcocoaprintersupport.dylib: # libcocoaprintersupport.dylib (compatibility version 0.0.0, current version 0.0.0) # /System/Library/Frameworks/AppKit.framework/Versions/C/AppKit (compatibility version 45.0.0, current version 1561.40.112) # @rpath/QtPrintSupport.framework/Versions/5/QtPrintSupport (compatibility version 5.11.0, current version 5.11.1) # .... CraftCore.log.debug("%s: ignoring library name id %s in %s", fileToFix, path, os.path.relpath(str(fileToFix), self.appPath)) continue if path.startswith("@executable_path/"): continue # already fixed if path.startswith("@rpath/"): # CraftCore.log.info("%s: can't handle @rpath library dep of yet: '%s'", fileToFix, path) CraftCore.log.debug("%s: can't handle @rpath library dep of yet: '%s'", fileToFix, path) # TODO: run otool -l and verify that we pick the right file? elif path.startswith("/usr/lib/") or path.startswith("/System/Library/Frameworks/"): CraftCore.log.debug("%s: allowing dependency on system library '%s'", fileToFix, path) elif path.startswith("/"): if not path.startswith(CraftStandardDirs.craftRoot()): CraftCore.log.error("%s: reference to absolute library path outside craftroot: %s", fileToFix, path) return False # file installed by craft -> bundle it into the .app if it doesn't exist yet if not self._addLibToAppImage(Path(path)): CraftCore.log.error(f"{fileToFix}: Failed to add library dependency '{path}' into bundle") return False if not self._updateLibraryReference(fileToFix, path): return False elif "/" not in path and path.startswith("lib"): # library reference without absolute path -> try to find the library # First check if it exists in Contents/Frameworks already guessedPath = Path(self.appPath, "Frameworks", path) if guessedPath.exists(): CraftCore.log.info("%s: relative library dependency is alreayd bundled: %s", fileToFix, guessedPath) else: guessedPath = Path(CraftStandardDirs.craftRoot(), "lib", path) if not guessedPath.exists(): CraftCore.log.error("%s: Could not find library dependency '%s' in craftroot", fileToFix, path) return False CraftCore.log.debug("%s: Found relative library reference %s in '%s'", fileToFix, path, guessedPath) if not self._addLibToAppImage(guessedPath): CraftCore.log.error("%s: Failed to add library dependency '%s' into bundle", fileToFix, guessedPath) return False if not self._updateLibraryReference(fileToFix, path): return False elif path.startswith("@loader_path/"): CraftCore.log.debug(f"{fileToFix}: Accept '{path}' into.") else: CraftCore.log.error("%s: don't know how to handle otool -L output: '%s'", fileToFix, path) return False return True def fixupAndBundleLibsRecursively(self, subdir: str): """Remove absolute references and budle all depedencies for all dylibs under :p subdir""" assert not subdir.startswith("/"), "Must be a relative path" for dirpath, dirs, files in os.walk(os.path.join(self.appPath, subdir)): for filename in files: fullpath = Path(dirpath, filename) if fullpath.is_symlink(): continue # No need to update symlinks since we will process the target eventually. if (filename.endswith(".so") or filename.endswith(".dylib") or ".so." in filename or (f"{fullpath.name}.framework" in str(fullpath) and utils.isBinary(str(fullpath)))): if not self.bundleLibraryDependencies(fullpath): CraftCore.log.info("Failed to bundle dependencies for '%s'", os.path.join(dirpath, filename)) return False return True def areLibraryDepsOkay(self, fullPath: Path): CraftCore.log.debug("Checking library dependencies of %s", fullPath) found_bad_lib = False libraryId = self._getLibraryNameId(fullPath) relativePath = os.path.relpath(str(fullPath), self.appPath) for dep in utils.getLibraryDeps(str(fullPath)): if dep == libraryId and not os.path.isabs(libraryId): continue # non-absolute library id is fine # @rpath and @executable_path is fine if dep.startswith("@rpath") or dep.startswith("@executable_path") or dep.startswith("@loader_path"): continue # Also allow /System/Library/Frameworks/ and /usr/lib: if dep.startswith("/usr/lib/") or dep.startswith("/System/Library/Frameworks/"): continue if dep.startswith(CraftStandardDirs.craftRoot()): CraftCore.log.error("ERROR: %s references absolute library path from craftroot: %s", relativePath, dep) elif dep.startswith("/"): CraftCore.log.error("ERROR: %s references absolute library path: %s", relativePath, dep) else: CraftCore.log.error("ERROR: %s has bad dependency: %s", relativePath, dep) found_bad_lib = True return not found_bad_lib def checkLibraryDepsRecursively(self, subdir: str): """Check that all absolute references and budle all depedencies for all dylibs under :p subdir""" assert not subdir.startswith("/"), "Must be a relative path" foundError = False for dirpath, dirs, files in os.walk(os.path.join(self.appPath, subdir)): for filename in files: fullpath = Path(dirpath, filename) if fullpath.is_symlink() and not fullpath.exists(): CraftCore.log.error("Found broken symlink '%s' (%s)", fullpath, os.readlink(str(fullpath))) foundError = True continue if filename.endswith(".so") or filename.endswith(".dylib") or ".so." in filename: if not self.areLibraryDepsOkay(fullpath): CraftCore.log.error("Found library dependency error in '%s'", fullpath) foundError = True return not foundError if __name__ == '__main__': print("Testing MacDMGPackager.py") defaultFile = CraftStandardDirs.craftRoot() + "/lib/libKF5TextEditor.5.dylib" sourceFile = defaultFile if len(sys.argv) else sys.argv[1] utils.system(["otool", "-L", sourceFile]) import tempfile with tempfile.TemporaryDirectory() as td: source = os.path.realpath(sourceFile) target = os.path.join(td, os.path.basename(source)) utils.copyFile(source, target, linkOnly=False) bundler = MacDylibBundler(td) bundler.bundleLibraryDependencies(Path(target)) print("Checked libs:", bundler.checkedLibs) utils.system(["find", td]) utils.system(["ls", "-laR", td]) if not bundler.areLibraryDepsOkay(Path(target)): print("Error") if not bundler.checkLibraryDepsRecursively("Contents/Frameworks"): print("Error 2") # utils.system(["find", td, "-type", "f", "-execdir", "otool", "-L", "{}", ";"]) diff --git a/bin/utils.py b/bin/utils.py index e2162693b..0ef6f9dac 100644 --- a/bin/utils.py +++ b/bin/utils.py @@ -1,1157 +1,1165 @@ # -*- coding: utf-8 -*- # copyright: # Holger Schroeder # Patrick Spendrin # Ralf Habacker # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import configparser import contextlib import glob import inspect import io import os import re import shlex import stat import shutil import sys import subprocess import tempfile from pathlib import Path import Notifier.NotificationLoader from Blueprints.CraftVersion import CraftVersion from CraftCore import CraftCore from CraftDebug import deprecated from CraftOS.osutils import OsUtils from CraftSetupHelper import SetupHelper from CraftStandardDirs import CraftStandardDirs from Utils import CraftChoicePrompt def abstract(): caller = inspect.getouterframes(inspect.currentframe())[1][3] raise NotImplementedError(caller + ' must be implemented in subclass') ### unpack functions def unpackFiles(downloaddir, filenames, workdir): """unpack (multiple) files specified by 'filenames' from 'downloaddir' into 'workdir'""" for filename in filenames: if (not unpackFile(downloaddir, filename, workdir)): return False return True def unpackFile(downloaddir, filename, workdir): """unpack file specified by 'filename' from 'downloaddir' into 'workdir'""" CraftCore.log.debug(f"unpacking this file: {filename}") if not filename: return True (shortname, ext) = os.path.splitext(filename) if ext == "": CraftCore.log.warning(f"unpackFile called on invalid file extension {filename}") return True if OsUtils.isWin() and not OsUtils.supportsSymlinks(): CraftCore.log.warning("Please enable Windows 10 development mode to enable support for symlinks.\n" "This will enable faster extractions.\n" "https://docs.microsoft.com/en-us/windows/uwp/get-started/enable-your-device-for-development") if CraftCore.cache.findApplication("7za"): # we use tar on linux not 7z, don't use tar on windows as it skips symlinks # test it with breeze-icons if (not OsUtils.isWin() or (OsUtils.supportsSymlinks() and CraftCore.cache.getVersion("7za", versionCommand="-version") >= "16") or not re.match("(.*\.tar.*$|.*\.tgz$)", filename)): return un7zip(os.path.join(downloaddir, filename), workdir, ext) try: shutil.unpack_archive(os.path.join(downloaddir, filename), workdir) except Exception as e: CraftCore.log.error(f"Failed to unpack {filename}", exc_info=e) return False return True def un7zip(fileName, destdir, flag=None): ciMode = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) createDir(destdir) kw = {} progressFlags = [] type = [] resolveSymlinks = False app = CraftCore.cache.findApplication("7za") if not ciMode and CraftCore.cache.checkCommandOutputFor(app, "-bs"): progressFlags = ["-bso2", "-bsp1"] kw["stderr"] = subprocess.PIPE if flag == ".7z": # Actually this is not needed for a normal archive. # But git is an exe file renamed to 7z and we need to specify the type. # Yes it is an ugly hack. type = ["-t7z"] if re.match("(.*\.tar.*$|.*\.tgz$)", fileName): if progressFlags: if ciMode: progressFlags = [] else: # print progress to stderr progressFlags = ["-bsp2"] kw["pipeProcess"] = subprocess.Popen([app, "x", fileName, "-so"] + progressFlags, stdout = subprocess.PIPE) if OsUtils.isWin(): resolveSymlinks = True if progressFlags: progressFlags = ["-bsp0"] command = [app, "x", "-si", f"-o{destdir}", "-ttar"] + progressFlags else: tar = CraftCore.cache.findApplication("tar") command = [tar, "--directory", destdir, "-xf", "-"] else: command = [app, "x", "-r", "-y", f"-o{destdir}", fileName] + type + progressFlags # While 7zip supports symlinks cmake 3.8.0 does not support symlinks return system(command, displayProgress=True, **kw) and (not resolveSymlinks or replaceSymlinksWithCopies(destdir)) def compress(archive : str, source : str) -> bool: ciMode = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) def __7z(archive, source): archive = Path(archive) app = CraftCore.cache.findApplication("7za") kw = {} flags = [] if archive.suffix in {".appxsym", ".appxupload"}: flags.append("-tzip") if not ciMode and CraftCore.cache.checkCommandOutputFor(app, "-bs"): flags += ["-bso2", "-bsp1"] kw["stderr"] = subprocess.PIPE if CraftCore.compiler.isUnix: tar = CraftCore.cache.findApplication("tar") kw["pipeProcess"] = subprocess.Popen([tar, "-cf", "-", "-C", source, ".",], stdout=subprocess.PIPE) command = [app, "a", "-si", archive] + flags else: command = [app, "a", "-r", archive] + flags if isinstance(source, list): command += source elif os.path.isfile(source): command += [source] else: command += [os.path.join(source, "*")] return system(command, displayProgress=True, **kw) def __xz(archive, source): command = ["tar", "-cJf", archive, "-C"] if os.path.isfile(source): command += [source] else: command += [source, "."] return system(command) createDir(os.path.dirname(archive)) if os.path.isfile(archive): deleteFile(archive) if CraftCore.compiler.isUnix and archive.endswith(".tar.xz"): return __xz(archive, source) else: return __7z(archive, source) def system(cmd, displayProgress=False, logCommand=True, acceptableExitCodes=None, **kw): """execute cmd in a shell. All keywords are passed to Popen. stdout and stderr might be changed depending on the chosen logging options.""" return systemWithoutShell(cmd, displayProgress=displayProgress, logCommand=logCommand, acceptableExitCodes=acceptableExitCodes, **kw) def systemWithoutShell(cmd, displayProgress=False, logCommand=True, pipeProcess=None, acceptableExitCodes=None, secretCommand=False, **kw): """execute cmd. All keywords are passed to Popen. stdout and stderr might be changed depending on the chosen logging options. When the parameter "displayProgress" is True, stdout won't be logged to allow the display of progress bars.""" environment = kw.get("env", os.environ) cwd = kw.get("cwd", os.getcwd()) # if the first argument is not an absolute path replace it with the full path to the application if isinstance(cmd, list): # allow to pass other types, like ints or Path cmd = [str(x) for x in cmd] if pipeProcess: pipeProcess.args = [str(x) for x in pipeProcess.args] arg0 = cmd[0] if not "shell" in kw: kw["shell"] = False else: if not "shell" in kw: # use shell, arg0 might end up with "/usr/bin/svn" => needs shell so it can be executed kw["shell"] = True arg0 = shlex.split(cmd, posix=not CraftCore.compiler.isWindows)[0] matchQuoted = re.match("^\"(.*)\"$", arg0) if matchQuoted: CraftCore.log.warning(f"Please don't pass quoted paths to systemWithoutShell, app={arg0}") if not os.path.isfile(arg0) and not matchQuoted: app = CraftCore.cache.findApplication(arg0) else: app = arg0 if app: if isinstance(cmd, list): cmd[0] = app elif not matchQuoted: cmd = cmd.replace(arg0, f"\"{app}\"", 1) else: app = arg0 if secretCommand: CraftCore.debug.print(f"securely executing command: {app}") else: if logCommand: _logCommand = "" if pipeProcess: _logCommand = "{0} | ".format(" ".join(pipeProcess.args)) _logCommand += " ".join(cmd) if isinstance(cmd, list) else cmd CraftCore.debug.print("executing command: {0}".format(_logCommand)) if pipeProcess: CraftCore.log.debug(f"executing command: {pipeProcess.args!r} | {cmd!r}") else: CraftCore.log.debug(f"executing command: {cmd!r}") CraftCore.log.debug(f"CWD: {cwd!r}") CraftCore.log.debug(f"displayProgress={displayProgress}") CraftCore.debug.logEnv(environment) if pipeProcess: kw["stdin"] = pipeProcess.stdout if not displayProgress or CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): stdout = kw.get('stdout', sys.stdout) if stdout == sys.stdout: kw['stderr'] = subprocess.STDOUT kw['stdout'] = subprocess.PIPE proc = subprocess.Popen(cmd, **kw) if pipeProcess: pipeProcess.stdout.close() for line in proc.stdout: if isinstance(stdout, io.TextIOWrapper): if CraftCore.debug.verbose() < 3: # don't print if we write the debug log to stdout anyhow stdout.buffer.write(line) stdout.flush() elif stdout == subprocess.DEVNULL: pass elif isinstance(stdout, io.StringIO): stdout.write(line.decode("UTF-8")) else: stdout.write(line) CraftCore.log.debug("{app}: {out}".format(app=app, out=line.rstrip())) else: proc = subprocess.Popen(cmd, **kw) if pipeProcess: pipeProcess.stdout.close() if proc.stderr: for line in proc.stderr: CraftCore.log.debug("{app}: {out}".format(app=app, out=line.rstrip())) proc.communicate() proc.wait() if acceptableExitCodes is None: ok = proc.returncode == 0 else: ok = proc.returncode in acceptableExitCodes if not ok: if not secretCommand: msg = f"Command {cmd} failed with exit code {proc.returncode}" if not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): CraftCore.log.debug(msg) else: CraftCore.log.info(msg) else: CraftCore.log.info(f"{app} failed with exit code {proc.returncode}") return ok def cleanDirectory(directory): CraftCore.log.debug("clean directory %s" % directory) if os.path.exists(directory): # don't delete containg directrory as it might be a symlink and replacing it with a folder # breaks the behaviour with os.scandir(directory) as scan: for f in scan: if f.is_dir(): if not OsUtils.rmDir(f.path, force=True): return False else: if not OsUtils.rm(f.path, force=True): return False return True else: return createDir(directory) def getVCSType(url): """ return the type of the vcs url """ if not url: return "" if isGitUrl(url): return "git" elif isSvnUrl(url): return "svn" elif url.startswith("[hg]"): return "hg" ## \todo complete more cvs access schemes elif url.find("pserver:") >= 0: return "cvs" else: return "" def isGitUrl(Url): """ this function returns true, if the Url given as parameter is a git url: it either starts with git:// or the first part before the first '|' ends with .git or if the url starts with the token [git] """ if Url.startswith('git://'): return True # split away branch and tags splitUrl = Url.split('|') if splitUrl[0].endswith(".git"): return True if Url.startswith("[git]"): return True return False def isSvnUrl(url): """ this function returns true, if the Url given as parameter is a svn url """ if url.startswith("[svn]"): return True elif url.find("://") == -1: return True elif url.find("svn:") >= 0 or url.find("https:") >= 0 or url.find("http:") >= 0: return True return False def splitVCSUrl(Url): """ this function splits up an url provided by Url into the server name, the path, a branch or tag; it will return a list with 3 strings according to the following scheme: git://servername/path.git|4.5branch|v4.5.1 will result in ['git://servername:path.git', '4.5branch', 'v4.5.1'] This also works for all other dvcs""" splitUrl = Url.split('|') if len(splitUrl) < 3: c = [x for x in splitUrl] for dummy in range(3 - len(splitUrl)): c.append('') else: c = splitUrl[0:3] return c def replaceVCSUrl(Url): """ this function should be used to replace the url of a server this comes in useful if you e.g. need to switch the server url for a push url on gitorious.org """ configfile = os.path.join(CraftStandardDirs.etcBlueprintDir(), "..", "crafthosts.conf") replacedict = dict() # FIXME handle svn/git usernames and settings with a distinct naming # todo WTF if (("General", "KDESVNUSERNAME") in CraftCore.settings and CraftCore.settings.get("General", "KDESVNUSERNAME") != "username"): replacedict["git://git.kde.org/"] = "git@git.kde.org:" if os.path.exists(configfile): config = configparser.ConfigParser() config.read(configfile) # add the default KDE stuff if the KDE username is set. for section in config.sections(): host = config.get(section, "host") replace = config.get(section, "replace") replacedict[host] = replace for host in list(replacedict.keys()): if not Url.find(host) == -1: Url = Url.replace(host, replacedict[host]) break return Url def createImportLibs(dll_name, basepath): """creating the import libraries for the other compiler(if ANSI-C libs)""" dst = os.path.join(basepath, "lib") if (not os.path.exists(dst)): os.mkdir(dst) # check whether the required binary tools exist HAVE_GENDEF = CraftCore.cache.findApplication("gendef") is not None USE_GENDEF = HAVE_GENDEF HAVE_LIB = CraftCore.cache.findApplication("lib") is not None HAVE_DLLTOOL = CraftCore.cache.findApplication("dlltool") is not None CraftCore.log.debug(f"gendef found: {HAVE_GENDEF}") CraftCore.log.debug(f"gendef used: {USE_GENDEF}") CraftCore.log.debug(f"lib found: {HAVE_LIB}") CraftCore.log.debug(f"dlltool found: {HAVE_DLLTOOL}") dllpath = os.path.join(basepath, "bin", "%s.dll" % dll_name) defpath = os.path.join(basepath, "lib", "%s.def" % dll_name) exppath = os.path.join(basepath, "lib", "%s.exp" % dll_name) imppath = os.path.join(basepath, "lib", "%s.lib" % dll_name) gccpath = os.path.join(basepath, "lib", "%s.dll.a" % dll_name) if not HAVE_GENDEF and os.path.exists(defpath): HAVE_GENDEF = True USE_GENDEF = False if not HAVE_GENDEF: CraftCore.log.warning("system does not have gendef.exe") return False if not HAVE_LIB and not os.path.isfile(imppath): CraftCore.log.warning("system does not have lib.exe (from msvc)") if not HAVE_DLLTOOL and not os.path.isfile(gccpath): CraftCore.log.warning("system does not have dlltool.exe") # create .def if USE_GENDEF: cmd = "gendef - %s -a > %s " % (dllpath, defpath) system(cmd) if (HAVE_LIB and not os.path.isfile(imppath)): # create .lib cmd = "lib /machine:x86 /def:%s /out:%s" % (defpath, imppath) system(cmd) if (HAVE_DLLTOOL and not os.path.isfile(gccpath)): # create .dll.a cmd = "dlltool -d %s -l %s -k" % (defpath, gccpath) system(cmd) if os.path.exists(defpath): os.remove(defpath) if os.path.exists(exppath): os.remove(exppath) return True def createSymlink(source, linkName, useAbsolutePath=False, targetIsDirectory=False): if not useAbsolutePath and os.path.isabs(linkName): srcPath = linkName srcPath = os.path.dirname(srcPath) source = os.path.relpath(source, srcPath) createDir(os.path.dirname(linkName)) CraftCore.log.debug(f"creating symlink: {linkName} -> {source}") try: os.symlink(source, linkName, targetIsDirectory) return True except Exception as e: CraftCore.log.warning(e) return False def createDir(path): """Recursive directory creation function. Makes all intermediate-level directories needed to contain the leaf directory""" if not os.path.lexists(path): CraftCore.log.debug(f"creating directory {path}") os.makedirs(path) return True def copyFile(src : Path, dest : Path, linkOnly=CraftCore.settings.getboolean("General", "UseHardlinks", False)): """ copy file from src to dest""" CraftCore.log.debug("copy file from %s to %s" % (src, dest)) src = Path(src) dest = Path(dest) if dest.is_dir(): dest = dest / src.name else: createDir(dest.parent) if os.path.lexists(dest): CraftCore.log.warning(f"Overriding:\t{dest} with\n\t\t{src}") if src == dest: CraftCore.log.error(f"Can't copy a file into itself {src}=={dest}") return False OsUtils.rm(dest, True) # don't link to links if linkOnly and not os.path.islink(src): try: os.link(src, dest) return True except: CraftCore.log.warning("Failed to create hardlink %s for %s" % (dest, src)) try: shutil.copy2(src, dest, follow_symlinks=False) except Exception as e: CraftCore.log.error(f"Failed to copy file:\n{src} to\n{dest}", exc_info=e) return False return True def copyDir(srcdir, destdir, linkOnly=CraftCore.settings.getboolean("General", "UseHardlinks", False), copiedFiles=None): """ copy directory from srcdir to destdir """ CraftCore.log.debug("copyDir called. srcdir: %s, destdir: %s" % (srcdir, destdir)) srcdir = Path(srcdir) if not srcdir.exists(): CraftCore.log.warning(f"copyDir called. srcdir: {srcdir} does not exists") return True destdir = Path(destdir) if not destdir.exists(): createDir(destdir) try: with os.scandir(srcdir) as scan: for entry in scan: dest = destdir / Path(entry.path).parent.relative_to(srcdir) / entry.name if entry.is_dir(): if entry.is_symlink(): # copy the symlinks without resolving them if not copyFile(entry.path, dest, linkOnly=False): return False if copiedFiles is not None: copiedFiles.append(str(dest)) else: if not copyDir(entry.path, dest, copiedFiles=copiedFiles, linkOnly=linkOnly): return False else: # symlinks to files are included in `files` if not copyFile(entry.path, dest,linkOnly=linkOnly): return False if copiedFiles is not None: copiedFiles.append(str(dest)) except Exception as e: CraftCore.log.error(f"Failed to copy dir:\n{srcdir} to\n{destdir}", exc_info=e) return False return True def globCopyDir(srcDir : str, destDir : str, pattern : [str], linkOnly=CraftCore.settings.getboolean("General", "UseHardlinks", False)) -> bool: files = [] for p in pattern: files.extend(glob.glob(os.path.join(srcDir, p), recursive=True)) for f in files: if not copyFile(f, os.path.join(destDir, os.path.relpath(f, srcDir)), linkOnly=linkOnly): return False return True def mergeTree(srcdir, destdir): """ moves directory from @p srcdir to @p destdir If a directory in @p destdir exists, just write into it """ srcdir = Path(srcdir) destdir = Path(destdir) if not createDir(destdir): return False CraftCore.log.debug(f"mergeTree called. srcdir: {srcdir}, destdir: {destdir}") if srcdir.samefile(destdir): CraftCore.log.critical(f"mergeTree called on the same directory srcdir: {srcdir}, destdir: {destdir}") return False with os.scandir(srcdir) as scan: for src in scan: dest = destdir / src.name if Path(src.path) in dest.parents: CraftCore.log.info(f"mergeTree: skipping moving of {src.path} to {dest}") continue if dest.exists(): if dest.is_dir(): if dest.is_symlink(): if dest.samefile(src): CraftCore.log.info(f"mergeTree: skipping moving of {src.path} to {dest} as a symlink with the same destination already exists") continue else: CraftCore.log.critical(f"mergeTree failed: {src.path} and {dest} are both symlinks but point to different folders") return False if src.is_symlink() and not dest.is_symlink(): CraftCore.log.critical(f"mergeTree failed: how to merge symlink {src.path} into {dest}") return False if not src.is_symlink() and dest.is_symlink(): CraftCore.log.critical(f"mergeTree failed: how to merge folder {src.path} into symlink {dest}") return False if not mergeTree(src.path, dest): return False else: CraftCore.log.critical(f"mergeTree failed: how to merge folder {src.path} into file {dest}\n" f"If this error occured during packaging, consider extending the blacklist.") return False else: if not moveFile(src.path, destdir): return False if not os.listdir(srcdir): # Cleanup (only removing empty folders) return rmtree(srcdir) else: # we move a directory in one of its sub directories assert srcdir in destdir.parents return True @deprecated("moveFile") def moveDir(srcdir, destdir): """ move directory from srcdir to destdir """ return moveFile(srcdir, destdir) def moveFile(src, dest): """move file from src to dest""" CraftCore.log.debug("move file from %s to %s" % (src, dest)) try: shutil.move(src, dest, copy_function=lambda src, dest, *kw : shutil.copy2(src, dest, *kw, follow_symlinks=False)) except Exception as e: CraftCore.log.warning(e) return False return True def rmtree(directory): """ recursively delete directory """ CraftCore.log.debug("rmtree called. directory: %s" % (directory)) try: shutil.rmtree(directory, True) # ignore errors except Exception as e: CraftCore.log.warning(e) return False return True def deleteFile(fileName): """delete file """ if not os.path.exists(fileName): return False CraftCore.log.debug("delete file %s " % (fileName)) try: os.remove(fileName) except Exception as e: CraftCore.log.warning(e) return False return True def putenv(name, value): """set environment variable""" if value is None: msg = f"unset environment variable -- unset {name}" if name in os.environ: del os.environ[name] else: msg = f"set environment variable -- set {name}={value}" os.environ[name] = value if CraftCore.settings.getboolean("CraftDebug", "PrintPutEnv", False): CraftCore.log.info(msg) else: CraftCore.log.debug(msg) return True def applyPatch(sourceDir, f, patchLevel='0'): """apply single patch""" if os.path.isdir(f): # apply a whole dir of patches for patch in os.listdir(f): if not applyPatch(sourceDir, os.path.join(f, patch), patchLevel): return False return True with tempfile.TemporaryDirectory() as tmp: # rewrite the patch, the gnu patch on Windows is only capable # to read \r\n patches tmpPatch = os.path.join(tmp, os.path.basename(f)) with open(f, "rt", encoding="utf-8") as p: patchContent = p.read() with open(tmpPatch, "wt", encoding="utf-8") as p: p.write(patchContent) cmd = ["patch", "--ignore-whitespace", "-d", sourceDir, "-p", str(patchLevel), "-i", tmpPatch] result = system(cmd) if not result: CraftCore.log.warning(f"applying {f} failed!") return result def embedManifest(executable, manifest): ''' Embed a manifest to an executable using either the free kdewin manifest if it exists in dev-utils/bin or the one provided by the Microsoft Platform SDK if it is installed' ''' if not os.path.isfile(executable) or not os.path.isfile(manifest): # We die here because this is a problem with the blueprint files CraftCore.log.critical("embedManifest %s or %s do not exist" % (executable, manifest)) CraftCore.log.debug("embedding ressource manifest %s into %s" % \ (manifest, executable)) return system(["mt", "-nologo", "-manifest", manifest, f"-outputresource:{executable};1"]) def notify(title, message, alertClass=None, log=True): if log: CraftCore.debug.step(f"{title}: {message}") backends = CraftCore.settings.get("General", "Notify", "") if CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) or backends == "": return backends = Notifier.NotificationLoader.load(backends.split(";")) for backend in backends.values(): backend.notify(title, message, alertClass) def levenshtein(s1, s2): if len(s1) < len(s2): return levenshtein(s2, s1) if not s1: return len(s2) previous_row = range(len(s2) + 1) for i, c1 in enumerate(s1): current_row = [i + 1] for j, c2 in enumerate(s2): insertions = previous_row[ j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer deletions = current_row[j] + 1 # than s2 substitutions = previous_row[j] + (c1 != c2) current_row.append(min(insertions, deletions, substitutions)) previous_row = current_row return previous_row[-1] def createShim(shim, target, args=None, guiApp=False, useAbsolutePath=False) -> bool: if not useAbsolutePath and os.path.isabs(target): target = os.path.relpath(target, os.path.dirname(shim)) createDir(os.path.dirname(shim)) if os.path.exists(shim): deleteFile(shim) if not args: args = [] elif isinstance(args, str): CraftCore.log.error("Please pass args as [str]") return system(f"kshimgen --create {shim} {target} -- {args}") return system(["kshimgen", "--create", shim, target , "--"] + args) def replaceSymlinksWithCopies(path, _replaceDirs=False): def resolveLink(path): while os.path.islink(path): toReplace = os.readlink(path) if not os.path.isabs(toReplace): path = os.path.join(os.path.dirname(path), toReplace) else: path = toReplace return path # symlinks to dirs are resolved after we resolved the files dirsToResolve = [] ok = True for root, _, files in os.walk(path): for svg in files: if not ok: return False path = os.path.join(root, svg) if os.path.islink(path): toReplace = resolveLink(path) if not os.path.exists(toReplace): CraftCore.log.error(f"Resolving {path} failed: {toReplace} does not exists.") continue if toReplace != path: if os.path.isdir(toReplace): if not _replaceDirs: dirsToResolve.append(path) else: os.unlink(path) ok = copyDir(toReplace, path) else: os.unlink(path) ok = copyFile(toReplace, path) while dirsToResolve: d = dirsToResolve.pop() if not os.path.exists(resolveLink(d)): CraftCore.log.warning(f"Delay replacement of {d}") dirsToResolve.append(d) continue if not replaceSymlinksWithCopies(os.path.dirname(d), _replaceDirs=True): return False return True def printProgress(percent): width, _ = shutil.get_terminal_size((80, 20)) width -= 20 # margin times = int(width / 100 * percent) sys.stdout.write( "\r[{progress}{space}]{percent}%".format(progress="#" * times, space=" " * (width - times), percent=percent)) sys.stdout.flush() class ScopedEnv(object): def __init__(self, env): self.oldEnv = {} for key, value in env.items(): self.oldEnv[key] = os.environ.get(key, None) putenv(key, value) def reset(self): for key, value in self.oldEnv.items(): putenv(key, value) def __enter__(self): return self def __exit__(self, exc_type, exc_value, trback): self.reset() def normalisePath(path): path = os.path.abspath(path) if OsUtils.isWin(): return path.replace("\\", "/") return path def configureFile(inFile : str, outFile : str, variables : dict) -> bool: CraftCore.log.debug(f"configureFile {inFile} -> {outFile}\n{variables}") configPatter = re.compile(r"@{([^{}]+)}") with open(inFile, "rt", encoding="UTF-8") as f: script = f.read() matches = configPatter.findall(script) if not matches: CraftCore.log.debug("Nothing to configure") return False while matches: for match in matches: val = variables.get(match, None) if val is None: linenUmber = 0 for line in script.split("\n"): if match in line: break linenUmber += 1 raise Exception(f"Failed to configure {inFile}: @{{{match}}} is not in variables\n" f"{linenUmber}:{line}") script = script.replace(f"@{{{match}}}", str(val)) matches = configPatter.findall(script) os.makedirs(os.path.dirname(outFile), exist_ok=True) with open(outFile, "wt", encoding="UTF-8") as f: f.write(script) return True def limitCommandLineLength(command : [str], args : [str]) -> [[str]]: # the actual limit is hard to get in python so lets just use a working random size SIZE = 1024 * 4 out = [] commandSize = sum(map(len, command)) if commandSize >= SIZE: CraftCore.log.error("Failed to compute command, command too long") return [] currentSize = commandSize tmp = [] for a in args: le = len(a) if currentSize + le >= SIZE: out.append(command + tmp) tmp = [] currentSize = commandSize tmp.append(a) currentSize += le if tmp: out.append(command + tmp) return out def sign(fileNames : [str]) -> bool: if not CraftCore.settings.getboolean("CodeSigning", "Enabled", False): return True if not CraftCore.compiler.isWindows: CraftCore.log.warning("Code signing is currently only supported on Windows") return True signTool = CraftCore.cache.findApplication("signtool", forceCache=True) if not signTool: env = SetupHelper.getMSVCEnv() signTool = CraftCore.cache.findApplication("signtool", env["PATH"], forceCache=True) if not signTool: CraftCore.log.warning("Code signing requires a VisualStudio installation") return False command = [signTool, "sign", "/tr", "http://timestamp.digicert.com", "/td", "SHA256", "/fd", "SHA256", "/a"] certFile = CraftCore.settings.get("CodeSigning", "Certificate", "") subjectName = CraftCore.settings.get("CodeSigning", "CommonName", "") certProtected = CraftCore.settings.getboolean("CodeSigning", "Protected", False) kwargs = dict() if certFile: command += ["/f", certFile] if subjectName: command += ["/n", subjectName] if certProtected: password = CraftChoicePrompt.promptForPassword(message='Enter the password for your package signing certificate', key="WINDOWS_CODE_SIGN_CERTIFICATE_PASSWORD") command += ["/p", password] kwargs["secretCommand"] = True if True or CraftCore.debug.verbose() > 0: command += ["/v"] else: command += ["/q"] for args in limitCommandLineLength(command, fileNames): if not system(args, **kwargs): return False return True def signMacApp(appPath : str): if not CraftCore.settings.getboolean("CodeSigning", "Enabled", False): return True devID = CraftCore.settings.get("CodeSigning", "MacDeveloperId") loginKeychain = CraftCore.settings.get("CodeSigning", "MacKeychainPath", os.path.expanduser("~/Library/Keychains/login.keychain")) if CraftCore.settings.getboolean("CodeSigning", "Protected", False): if not unlockMacKeychain(loginKeychain): return False # Recursively sign app if not system(["codesign", "--keychain", loginKeychain, "--sign", f"Developer ID Application: {devID}", "--force", "--preserve-metadata=entitlements", "--options", "runtime", "--verbose=99", "--deep", appPath]): return False ## Verify signature if not system(["codesign", "--display", "--verbose", appPath]): return False if not system(["codesign", "--verify", "--verbose", "--strict", appPath]): return False # TODO: this step might require notarisation system(["spctl", "-a", "-t", "exec", "-vv", appPath]) ## Validate that the key used for signing the binary matches the expected TeamIdentifier ## needed to pass the SocketApi through the sandbox #if not utils.system("codesign -dv %s 2>&1 | grep 'TeamIdentifier=%s'" % (self.appPath, teamIdentifierFromConfig)): #return False return True def signMacPackage(packagePath : str): packagePath = Path(packagePath) if not CraftCore.settings.getboolean("CodeSigning", "Enabled", False): return True devID = CraftCore.settings.get("CodeSigning", "MacDeveloperId") loginKeychain = CraftCore.settings.get("CodeSigning", "MacKeychainPath", os.path.expanduser("~/Library/Keychains/login.keychain")) if CraftCore.settings.getboolean("CodeSigning", "Protected", False): if not unlockMacKeychain(loginKeychain): return False if packagePath.name.endswith(".dmg"): # sign dmg if not system(["codesign", "--force", "--keychain", loginKeychain, "--sign", f"Developer ID Application: {devID}", packagePath]): return False # TODO: this step would require notarisation # verify dmg signature system(["spctl", "-a", "-t", "open", "--context", "context:primary-signature", packagePath]) else: # sign pkg packagePathTmp = f"{packagePath}.sign" if not system(["productsign", "--keychain", loginKeychain, "--sign", f"Developer ID Installer: {devID}", packagePath, packagePathTmp]): return False os.rename(packagePathTmp, packagePath) return True def unlockMacKeychain(loginKeychain : str): password = CraftChoicePrompt.promptForPassword(message='Enter the password for your package signing certificate', key="MAC_KEYCHAIN_PASSWORD") if not system(["security", "set-key-partition-list", "-S", "apple-tool:,apple:,codesign:", "-s" ,"-k", password, loginKeychain], stdout=subprocess.DEVNULL, secretCommand=True): CraftCore.log.error("Failed to unlock keychain.") return False return True def isExecuatable(fileName : Path): fileName = Path(fileName) if CraftCore.compiler.isWindows: return fileName.suffix in os.environ["PATHEXT"] return os.access(fileName, os.X_OK) def isBinary(fileName : str) -> bool: # https://en.wikipedia.org/wiki/List_of_file_signatures fileName = Path(fileName) MACH_O_64 = b"\xCF\xFA\xED\xFE" ELF = b"\x7F\x45\x4C\x46" if fileName.is_symlink() or fileName.is_dir(): return False if CraftCore.compiler.isWindows: if fileName.suffix in {".dll", ".exe"}: return True else: if CraftCore.compiler.isMacOS and ".dSYM/" in str(fileName): return False if fileName.suffix in {".so", ".dylib"}: return True elif isExecuatable(fileName): if CraftCore.compiler.isMacOS: signature = MACH_O_64 elif CraftCore.compiler.isLinux or CraftCore.compiler.isFreeBSD: signature = ELF else: raise Exception("Unsupported platform") with open(fileName, "rb") as f: return f.read(len(signature)) == signature return False def getLibraryDeps(path): deps = [] if CraftCore.compiler.isMacOS: # based on https://github.com/qt/qttools/blob/5.11/src/macdeployqt/shared/shared.cpp infoRe = re.compile("^\\t(.+) \\(compatibility version (\\d+\\.\\d+\\.\\d+), "+ "current version (\\d+\\.\\d+\\.\\d+)\\)$") with io.StringIO() as log: if not system(["otool", "-L", path], stdout=log, logCommand=False): return [] lines = log.getvalue().strip().split("\n") lines.pop(0)# name of the library for line in lines: match = infoRe.match(line) if match: deps.append(match[1]) return deps def regexFileFilter(filename : os.DirEntry, root : str, pattern : [re]=None) -> bool: """ return False if file does not match pattern""" # use linux style seperators relFilePath = Path(filename.path).relative_to(root).as_posix() for pattern in pattern: if pattern.search(relFilePath): CraftCore.log.debug(f"regExDirFilter: {relFilePath} matches: {pattern.pattern}") return True return False def filterDirectoryContent(root, whitelist=lambda f, root: True, blacklist=lambda g, root: False, allowBadSymlinks=False, handleAppBundleAsFile=False): """ Traverse through a directory tree and return every filename that the function whitelist returns as true and which do not match blacklist entries """ if not os.path.exists(root): return dirs = [root] while dirs: path = dirs.pop() with os.scandir(path) as scan: for filePath in scan: if not allowBadSymlinks and filePath.is_symlink(): if Path(root) not in Path(filePath.path).resolve().parents: CraftCore.log.debug(f"filterDirectoryContent: skipping {filePath.path}, it is not located under {root}") continue if filePath.is_dir(follow_symlinks=False): # handle .app folders and dsym as files if CraftCore.compiler.isMacOS: suffixes = {".dSYM"} if handleAppBundleAsFile: suffixes.update({".app", ".framework"}) if Path(filePath.path).suffix not in suffixes: dirs.append(filePath.path) continue else: dirs.append(filePath.path) continue if blacklist(filePath, root=root) and not whitelist(filePath, root=root): continue elif filePath.is_dir(): yield filePath.path elif filePath.is_file(): yield filePath.path elif filePath.is_symlink(): if not allowBadSymlinks: CraftCore.log.warning(f"{filePath.path} is an invalid link ({os.readlink(filePath.path)})") continue else: yield filePath.path else: CraftCore.log.warning(f"Unhandled case: {filePath}") raise Exception(f"Unhandled case: {filePath}") -@contextlib.contextmanager -def makeWritable(targetPath: Path): - if isinstance(targetPath, str): - targetPath = Path(targetPath) +def makeWritable(targetPath: Path) -> (bool, int): + """ Make a file writable if needed. Returns if the mode was changed and the curent mode of the file""" + targetPath = Path(targetPath) originalMode = targetPath.stat().st_mode - wasWritable = bool(originalMode & stat.S_IWUSR) + if not bool(originalMode & stat.S_IWUSR): + newMode = originalMode | stat.S_IWUSR + targetPath.chmod(newMode) + return (True, newMode) + return (False, originalMode) + +@contextlib.contextmanager +def makeTemporaryWritable(targetPath: Path): + targetPath = Path(targetPath) + wasReadOnly = False + mode = 0 try: # ensure it is writable - if not wasWritable: - targetPath.chmod(originalMode | stat.S_IWUSR) + wasReadOnly, mode = makeWritable(targetPath) yield targetPath finally: - if not wasWritable: - targetPath.chmod(originalMode) + if wasReadOnly: + targetPath.chmod(mode & ~stat.S_IWUSR) def getPDBForBinary(path :str) -> str: with open(path, "rb") as f: data = f.read() pdb = data.rfind(b".pdb") if pdb: return data[data.rfind(0x00, 0, pdb) + 1:pdb + 4].decode("utf-8") return "" def installShortcut(name : str, path : str, workingDir : str, icon : str, desciption : str): if not CraftCore.compiler.isWindows: return True from shells import Powershell pwsh = Powershell() shortcutPath = Path(os.environ["APPDATA"]) / f"Microsoft/Windows/Start Menu/Programs/Craft/{name}.lnk" shortcutPath.parent.mkdir(parents=True, exist_ok=True) return pwsh.execute([os.path.join(CraftCore.standardDirs.craftBin(), "install-lnk.ps1"), "-Path", pwsh.quote(path), "-WorkingDirectory", pwsh.quote(OsUtils.toNativePath(workingDir)), "-Name", pwsh.quote(shortcutPath), "-Icon", pwsh.quote(icon), "-Description", pwsh.quote(desciption)]) def strip(fileName): """strip debugging informations from shared libraries and executables - mingw only!!! """ if CraftCore.compiler.isMSVC() or not CraftCore.compiler.isGCCLike(): CraftCore.log.warning(f"Skipping stripping of {fileName} -- either disabled or unsupported with this compiler") return True fileName = Path(fileName) isBundle = False if CraftCore.compiler.isMacOS: bundleDir = list(filter(lambda x: x.name.endswith(".framework") or x.name.endswith(".app"), fileName.parents)) if bundleDir: suffix = "" # if we are a .app in a .framework we put the smbols in the same location if len(bundleDir) > 1: suffix = f"-{'.'.join([x.name for x in reversed(bundleDir[0:-1])])}" isBundle = True symFile = Path(f"{bundleDir[-1]}{suffix}.dSYM") else: symFile = Path(f"{fileName}.dSYM") else: symFile = Path(f"{fileName}.sym") if not isBundle and symFile.exists(): return True elif (symFile / "Contents/Resources/DWARF" / fileName.name).exists(): return True if CraftCore.compiler.isMacOS: return (system(["dsymutil", fileName, "-o", symFile]) and system(["strip", "-x", "-S", fileName])) else: return (system(["objcopy", "--only-keep-debug", fileName, symFile]) and system(["strip", "--strip-debug", "--strip-unneeded", fileName]) and system(["objcopy", "--add-gnu-debuglink", symFile, fileName]))