diff --git a/bin/Packager/MacDMGPackager.py b/bin/Packager/MacDMGPackager.py index ef1691ed4..077e53c64 100644 --- a/bin/Packager/MacDMGPackager.py +++ b/bin/Packager/MacDMGPackager.py @@ -1,370 +1,369 @@ from Packager.CollectionPackagerBase import * from Blueprints.CraftPackageObject import CraftPackageObject from Utils import CraftHash from pathlib import Path import contextlib import io import subprocess import stat import glob class MacDMGPackager( CollectionPackagerBase ): @InitGuard.init_once def __init__(self, whitelists=None, blacklists=None): CollectionPackagerBase.__init__(self, whitelists, blacklists) def createPackage(self): """ create a package """ CraftCore.log.debug("packaging using the MacDMGPackager") packageSymbols = CraftCore.settings.getboolean("Packager", "PackageDebugSymbols", False) if not self.internalCreatePackage(seperateSymbolFiles=packageSymbols): return False defines = self.setDefaults(self.defines) - - archive = os.path.normpath(self.archiveDir()) appPath = self.getMacAppPath(defines) + archive = os.path.normpath(self.archiveDir()) CraftCore.log.info(f"Packaging {appPath}") targetLibdir = os.path.join(appPath, "Contents", "Frameworks") utils.createDir(targetLibdir) moveTargets = [ (os.path.join(archive, "lib", "plugins"), os.path.join(appPath, "Contents", "PlugIns")), (os.path.join(archive, "plugins"), os.path.join(appPath, "Contents", "PlugIns")), (os.path.join(archive, "lib"), targetLibdir), (os.path.join(archive, "share"), os.path.join(appPath, "Contents", "Resources"))] if not appPath.startswith(archive): moveTargets += [(os.path.join(archive, "bin"), os.path.join(appPath, "Contents", "MacOS"))] for src, dest in moveTargets: if os.path.exists(src): if not utils.mergeTree(src, dest): return False dylibbundler = MacDylibBundler(appPath) with utils.ScopedEnv({'DYLD_FALLBACK_LIBRARY_PATH': targetLibdir + ":" + os.path.join(CraftStandardDirs.craftRoot(), "lib")}): CraftCore.log.info("Bundling main binary dependencies...") mainBinary = Path(appPath, "Contents", "MacOS", defines['appname']) if not dylibbundler.bundleLibraryDependencies(mainBinary): return False # Fix up the library dependencies of files in Contents/Frameworks/ CraftCore.log.info("Bundling library dependencies...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/Frameworks"): return False CraftCore.log.info("Bundling plugin dependencies...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/PlugIns"): return False if not utils.system(["macdeployqt", appPath, "-always-overwrite", "-verbose=1"]): return False # macdeployqt might just have added some explicitly blacklisted files blackList = Path(self.packageDir(), "mac_blacklist.txt") if blackList.exists(): pattern = [self.read_blacklist(str(blackList))] # use it as whitelist as we want only matches, ignore all others matches = utils.filterDirectoryContent(appPath, whitelist=lambda x, root: utils.regexFileFilter(x, root, pattern), blacklist=lambda x, root:True) for f in matches: CraftCore.log.info(f"Remove blacklisted file: {f}") utils.deleteFile(f) # macdeployqt adds some more plugins so we fix the plugins after calling macdeployqt dylibbundler.checkedLibs = set() # ensure we check all libs again (but # we should not need to make any changes) CraftCore.log.info("Fixing plugin dependencies after macdeployqt...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/PlugIns"): return False CraftCore.log.info("Fixing library dependencies after macdeployqt...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/Frameworks"): return False # Finally sanity check that we don't depend on absolute paths from the builder CraftCore.log.info("Checking for absolute library paths in package...") found_bad_dylib = False # Don't exit immeditately so that we log all the bad libraries before failing: if not dylibbundler.areLibraryDepsOkay(mainBinary): found_bad_dylib = True CraftCore.log.error("Found bad library dependency in main binary %s", mainBinary) if not dylibbundler.checkLibraryDepsRecursively("Contents/Frameworks"): CraftCore.log.error("Found bad library dependency in bundled libraries") found_bad_dylib = True if not dylibbundler.checkLibraryDepsRecursively("Contents/PlugIns"): CraftCore.log.error("Found bad library dependency in bundled plugins") found_bad_dylib = True if found_bad_dylib: CraftCore.log.error("Cannot not create .dmg since the .app contains a bad library depenency!") return False name = self.binaryArchiveName(fileType="", includeRevision=True) dmgDest = os.path.join(self.packageDestinationDir(), f"{name}.dmg") if os.path.exists(dmgDest): utils.deleteFile(dmgDest) appName = defines['appname'] + ".app" if not utils.system(["create-dmg", "--volname", name, # Add a drop link to /Applications: "--icon", appName, "140", "150", "--app-drop-link", "350", "150", dmgDest, appPath]): return False CraftHash.createDigestFiles(dmgDest) return True class MacDylibBundler(object): """ Bundle all .dylib files that are not provided by the system with the .app """ def __init__(self, appPath: str): # Avoid processing the same file more than once self.checkedLibs = set() self.appPath = appPath def _addLibToAppImage(self, libPath: Path) -> bool: assert libPath.is_absolute(), libPath libBasename = libPath.name targetPath = Path(self.appPath, "Contents/Frameworks/", libBasename) if targetPath.exists() and targetPath in self.checkedLibs: return True # Handle symlinks (such as libgit2.27.dylib -> libgit2.0.27.4.dylib): if libPath.is_symlink(): linkTarget = os.readlink(str(libPath)) CraftCore.log.info("Library dependency %s is a symlink to '%s'", libPath, linkTarget) if os.path.isabs(linkTarget): CraftCore.log.error("%s: Cannot handle absolute symlinks: '%s'", libPath, linkTarget) return False if ".." in linkTarget: CraftCore.log.error("%s: Cannot handle symlinks containing '..': '%s'", libPath, linkTarget) return False if libPath.resolve().parent != libPath.parent.resolve(): CraftCore.log.error("%s: Cannot handle symlinks to other directories: '%s' (%s vs %s)", libPath, linkTarget, libPath.resolve().parent, libPath.parent.resolve()) return False # copy the symlink and add the real file: utils.copyFile(str(libPath), str(targetPath), linkOnly=False) CraftCore.log.info("Added symlink '%s' (%s) to bundle -> %s", libPath, os.readlink(str(targetPath)), targetPath) self.checkedLibs.add(targetPath) symlinkTarget = libPath.with_name(os.path.basename(linkTarget)) CraftCore.log.info("Processing symlink target '%s'", symlinkTarget) if not self._addLibToAppImage(symlinkTarget): self.checkedLibs.remove(targetPath) return False # If the symlink target was processed, the symlink itself is also fine return True if not libPath.exists(): CraftCore.log.error("Library dependency '%s' does not exist", libPath) return False CraftCore.log.debug("Handling library dependency '%s'", libPath) if not targetPath.exists(): utils.copyFile(str(libPath), str(targetPath), linkOnly=False) CraftCore.log.info("Added library dependency '%s' to bundle -> %s", libPath, targetPath) if not self._fixupLibraryId(targetPath): return False for path in utils.getLibraryDeps(str(targetPath)): # check there aren't any references to the original location: if path == str(libPath): CraftCore.log.error("%s: failed to fix reference to original location for '%s'", targetPath, path) return False if not self.bundleLibraryDependencies(targetPath): CraftCore.log.error("%s: UNKNOWN ERROR adding '%s' into bundle", targetPath, libPath) return False if not os.path.exists(targetPath): CraftCore.log.error("%s: Library dependency '%s' doesn't exist after copying... Symlink error?", targetPath, libPath) return False self.checkedLibs.add(targetPath) return True @staticmethod def _updateLibraryReference(fileToFix: Path, oldRef: str, newRef: str = None) -> bool: if newRef is None: newRef = "@executable_path/../Frameworks/" + os.path.basename(oldRef) with utils.makeWritable(fileToFix): if not utils.system(["install_name_tool", "-change", oldRef, newRef, str(fileToFix)], logCommand=False): CraftCore.log.error("%s: failed to update library dependency path from '%s' to '%s'", fileToFix, oldRef, newRef) return False return True @staticmethod def _getLibraryNameId(fileToFix: Path) -> str: libraryIdOutput = io.StringIO( subprocess.check_output(["otool", "-D", str(fileToFix)]).decode("utf-8").strip()) lines = libraryIdOutput.readlines() if len(lines) == 1: return "" # Should have exactly one line with the id now assert len(lines) == 2, lines return lines[1].strip() @classmethod def _fixupLibraryId(cls, fileToFix: Path): libraryId = cls._getLibraryNameId(fileToFix) if libraryId and os.path.isabs(libraryId): CraftCore.log.debug("Fixing library id name for %s", libraryId) with utils.makeWritable(fileToFix): if not utils.system(["install_name_tool", "-id", os.path.basename(libraryId), str(fileToFix)], logCommand=False): CraftCore.log.error("%s: failed to fix absolute library id name for", fileToFix) return False return True def bundleLibraryDependencies(self, fileToFix: Path) -> bool: assert not fileToFix.is_symlink(), fileToFix if fileToFix.stat().st_nlink > 1: CraftCore.log.error("More than one hard link to library %s found! " "This might modify another accidentally.", fileToFix) CraftCore.log.info("Fixing library dependencies for %s", fileToFix) if not self._fixupLibraryId(fileToFix): return False # Ensure we have the current library ID since we need to skip it in the otool -L output libraryId = self._getLibraryNameId(fileToFix) for path in utils.getLibraryDeps(str(fileToFix)): if path == libraryId: # The first line of the otool output is (usually?) the library itself: # $ otool -L PlugIns/printsupport/libcocoaprintersupport.dylib: # PlugIns/printsupport/libcocoaprintersupport.dylib: # libcocoaprintersupport.dylib (compatibility version 0.0.0, current version 0.0.0) # /System/Library/Frameworks/AppKit.framework/Versions/C/AppKit (compatibility version 45.0.0, current version 1561.40.112) # @rpath/QtPrintSupport.framework/Versions/5/QtPrintSupport (compatibility version 5.11.0, current version 5.11.1) # .... CraftCore.log.debug("%s: ignoring library name id %s in %s", fileToFix, path, os.path.relpath(str(fileToFix), self.appPath)) continue if path.startswith("@executable_path/"): continue # already fixed if path.startswith("@rpath/"): # CraftCore.log.info("%s: can't handle @rpath library dep of yet: '%s'", fileToFix, path) CraftCore.log.debug("%s: can't handle @rpath library dep of yet: '%s'", fileToFix, path) # TODO: run otool -l and verify that we pick the right file? elif path.startswith("/usr/lib/") or path.startswith("/System/Library/Frameworks/"): CraftCore.log.debug("%s: allowing dependency on system library '%s'", fileToFix, path) elif path.startswith("@loader_path/"): # TODO: we don't set @loader_path anymore so lets remove this after the next cache rebuild. 24.09.2018 if not self._updateLibraryReference(fileToFix, path): return False elif path.startswith("/"): if not path.startswith(CraftStandardDirs.craftRoot()): # TODO: should this be an error? CraftCore.log.warning("%s: reference to absolute library path outside craftroot: %s", fileToFix, path) # return False # file installed by craft -> bundle it into the .app if it doesn't exist yet if not self._addLibToAppImage(Path(path)): CraftCore.log.error("%s: Failed to add library dependency '%s' into bundle", fileToFix, path) return False if not self._updateLibraryReference(fileToFix, path): return False elif "/" not in path and path.startswith("lib"): # library reference without absolute path -> try to find the library # First check if it exists in Contents/Frameworks already guessedPath = Path(self.appPath, "Frameworks", path) if guessedPath.exists(): CraftCore.log.info("%s: relative library dependency is alreayd bundled: %s", fileToFix, guessedPath) else: guessedPath = Path(CraftStandardDirs.craftRoot(), "lib", path) if not guessedPath.exists(): CraftCore.log.error("%s: Could not find library dependency '%s' in craftroot", fileToFix, path) return False CraftCore.log.debug("%s: Found relative library reference %s in '%s'", fileToFix, path, guessedPath) if not self._addLibToAppImage(guessedPath): CraftCore.log.error("%s: Failed to add library dependency '%s' into bundle", fileToFix, guessedPath) return False if not self._updateLibraryReference(fileToFix, path): return False else: CraftCore.log.error("%s: don't know how to handle otool -L output: '%s'", fileToFix, path) return False return True def fixupAndBundleLibsRecursively(self, subdir: str): """Remove absolute references and budle all depedencies for all dylibs under :p subdir""" assert not subdir.startswith("/"), "Must be a relative path" for dirpath, dirs, files in os.walk(os.path.join(self.appPath, subdir)): for filename in files: fullpath = Path(dirpath, filename) if fullpath.is_symlink(): continue # No need to update symlinks since we will process the target eventually. if (filename.endswith(".so") or filename.endswith(".dylib") or ".so." in filename or (f"{fullpath.name}.framework" in str(fullpath) and utils.isBinary(str(fullpath)))): if not self.bundleLibraryDependencies(fullpath): CraftCore.log.info("Failed to bundle dependencies for '%s'", os.path.join(dirpath, filename)) return False return True def areLibraryDepsOkay(self, fullPath: Path): CraftCore.log.debug("Checking library dependencies of %s", fullPath) found_bad_lib = False libraryId = self._getLibraryNameId(fullPath) relativePath = os.path.relpath(str(fullPath), self.appPath) for dep in utils.getLibraryDeps(str(fullPath)): if dep == libraryId and not os.path.isabs(libraryId): continue # non-absolute library id is fine # @rpath and @executable_path is fine if dep.startswith("@rpath") or dep.startswith("@executable_path"): continue # Also allow /System/Library/Frameworks/ and /usr/lib: if dep.startswith("/usr/lib/") or dep.startswith("/System/Library/Frameworks/"): continue if dep.startswith(CraftStandardDirs.craftRoot()): CraftCore.log.error("ERROR: %s references absolute library path from craftroot: %s", relativePath, dep) elif dep.startswith("/"): CraftCore.log.error("ERROR: %s references absolute library path: %s", relativePath, dep) else: CraftCore.log.error("ERROR: %s has bad dependency: %s", relativePath, dep) found_bad_lib = True return not found_bad_lib def checkLibraryDepsRecursively(self, subdir: str): """Check that all absolute references and budle all depedencies for all dylibs under :p subdir""" assert not subdir.startswith("/"), "Must be a relative path" foundError = False for dirpath, dirs, files in os.walk(os.path.join(self.appPath, subdir)): for filename in files: fullpath = Path(dirpath, filename) if fullpath.is_symlink() and not fullpath.exists(): CraftCore.log.error("Found broken symlink '%s' (%s)", fullpath, os.readlink(str(fullpath))) foundError = True continue if filename.endswith(".so") or filename.endswith(".dylib") or ".so." in filename: if not self.areLibraryDepsOkay(fullpath): CraftCore.log.error("Found library dependency error in '%s'", fullpath) foundError = True return not foundError if __name__ == '__main__': print("Testing MacDMGPackager.py") defaultFile = CraftStandardDirs.craftRoot() + "/lib/libKF5TextEditor.5.dylib" sourceFile = defaultFile if len(sys.argv) else sys.argv[1] utils.system(["otool", "-L", sourceFile]) import tempfile with tempfile.TemporaryDirectory() as td: source = os.path.realpath(sourceFile) target = os.path.join(td, os.path.basename(source)) utils.copyFile(source, target, linkOnly=False) bundler = MacDylibBundler(td) bundler.bundleLibraryDependencies(Path(target)) print("Checked libs:", bundler.checkedLibs) utils.system(["find", td]) utils.system(["ls", "-laR", td]) if not bundler.areLibraryDepsOkay(Path(target)): print("Error") if not bundler.checkLibraryDepsRecursively("Contents/Frameworks"): print("Error 2") # utils.system(["find", td, "-type", "f", "-execdir", "otool", "-L", "{}", ";"]) diff --git a/bin/shells.py b/bin/shells.py index f1b479a0d..a31d1b03e 100644 --- a/bin/shells.py +++ b/bin/shells.py @@ -1,208 +1,208 @@ #!/usr/bin/env python """ provides shells """ import platform import subprocess import sys from CraftCore import CraftCore from Blueprints.CraftVersion import CraftVersion from CraftOS.osutils import OsUtils import utils import os import shutil class BashShell(object): def __init__(self): self._environment = {} self._useMSVCCompatEnv = False @property def useMSVCCompatEnv(self): return self._useMSVCCompatEnv @useMSVCCompatEnv.setter def useMSVCCompatEnv(self, b): self._useMSVCCompatEnv = b self._environment = {} @property def environment(self): if not self._environment: mergeroot = self.toNativePath(CraftCore.standardDirs.craftRoot()) ldflags = f" -L{mergeroot}/lib " cflags = f" -I{mergeroot}/include " if CraftCore.compiler.isMacOS: # Only look for includes/libraries in the XCode SDK on MacOS to avoid errors with # libraries installed by homebrew (causes errors e.g. with iconv since headers will be # found in /usr/local/include first but libraries are searched for in /usr/lib before # /usr/local/lib. See https://langui.sh/2015/07/24/osx-clang-include-lib-search-paths/ if CraftCore.compiler.macUseSDK: sdkPath = CraftCore.cache.getCommandOutput("xcrun", "--show-sdk-path")[1].strip() deploymentFlag = "-mmacosx-version-min=" + CraftCore.compiler.macOSDeploymentTarget cflags = f" -isysroot {sdkPath} {deploymentFlag}" + cflags # See https://github.com/Homebrew/homebrew-core/issues/2674 for the -no_weak_imports flag ldflags = f" -isysroot {sdkPath} {deploymentFlag} -Wl,-no_weak_imports" + ldflags # Note: MACOSX_DEPLOYMENT_TARGET is set in utils.system() so doesn't need to be set here else: # Ensure that /usr/include comes before /usr/local/include in the header search path to avoid # pulling in headers from /usr/local/include (e.g. installed by homebrew) that will cause # linker errors later. cflags = " -isystem /usr/include " + cflags if CraftCore.compiler.isMSVC(): # based on Windows-MSVC.cmake if self.buildType == "Release": cflags += " -MD -O2 -Ob2 -DNDEBUG " elif self.buildType == "RelWithDebInfo": cflags += " -MD -Zi -O2 -Ob1 -DNDEBUG " ldflags += " -debug " elif self.buildType == "Debug": cflags += " -MDd -Zi -Ob0 -Od " ldflags += " -debug -pdbtype:sept " else: if self.buildType == "Release": cflags += " -O3 -DNDEBUG " if self.buildType == "RelWithDebInfo": cflags += " -O2 -g -DNDEBUG " elif self.buildType == "Debug": cflags += " -O0 -g3 " if OsUtils.isWin(): def convertPath(path : str): return ":".join([self.toNativePath(p) for p in path.split(os.path.pathsep)]) path = "/usr/local/bin:/usr/bin:/bin:/usr/bin/site_perl:/usr/bin/vendor_perl:/usr/bin/core_perl" if CraftCore.compiler.isMinGW(): gcc = shutil.which("gcc") if gcc: path = f"{self.toNativePath(os.path.dirname(gcc))}:{path}" self._environment["PATH"] = f"{path}:{convertPath(os.environ['PATH'])}" self._environment["PKG_CONFIG_PATH"] = convertPath(os.environ["PKG_CONFIG_PATH"]) if "make" in self._environment: del self._environment["make"] # MSYSTEM is used by uname if CraftCore.compiler.isMinGW(): self._environment["MSYSTEM"] = f"MINGW{CraftCore.compiler.bits}_CRAFT" elif CraftCore.compiler.isMSVC(): self._environment["MSYSTEM"] = f"MSYS{CraftCore.compiler.bits}_CRAFT" if self.useMSVCCompatEnv and CraftCore.compiler.isMSVC(): automake = [] for d in os.scandir(os.path.join(os.path.dirname(self._findBash()), "..", "share")): if d.name.startswith("automake"): automake += [(d.name.rsplit("-")[1], os.path.realpath(d.path))] automake.sort(key=lambda x: CraftVersion(x[0])) latestAutomake = automake[-1][1] if False: cl = "clang-cl" else: cl = "cl" clWrapper = self.toNativePath(os.path.join(latestAutomake, "compile")) self._environment["LD"] = "link -nologo" self._environment["CC"] = f"{clWrapper} {cl} -nologo" self._environment["CXX"] = self._environment["CC"] self._environment["CPP"] = f"{cl} -nologo -EP" self._environment["CXXCPP"] = self._environment["CPP"] self._environment["NM"] = "dumpbin -symbols" self._environment["RC"] = f"windres -O COFF --target={'pe-i386' if CraftCore.compiler.isX86() else 'pe-x86-64'} --preprocessor='cl -nologo -EP -DRC_INVOKED -DWINAPI_FAMILY=0'" self._environment["STRIP"] = ":" self._environment["RANLIB"] = ":" self._environment["F77"] = "no" self._environment["FC"] = "no" cflags += (" -GR -W3 -EHsc" # dynamic and exceptions enabled " -D_USE_MATH_DEFINES -DWIN32_LEAN_AND_MEAN -DNOMINMAX -D_CRT_SECURE_NO_WARNINGS" " -wd4005" # don't warn on redefine " -wd4996" # The POSIX name for this item is deprecated. ) if CraftCore.compiler.getMsvcPlatformToolset() > 120: cflags += " -FS" self._environment["CFLAGS"] = os.environ.get("CFLAGS", "").replace("$", "$$") + cflags self._environment["CXXFLAGS"] = os.environ.get("CXXFLAGS", "").replace("$", "$$") + cflags self._environment["LDFLAGS"] = os.environ.get("LDFLAGS", "").replace("$", "$$") + ldflags return self._environment @property def buildType(self): return CraftCore.settings.get("Compile", "BuildType", "RelWithDebInfo") @staticmethod def toNativePath(path): if OsUtils.isWin(): return OsUtils.toMSysPath(path) else: return path def _findBash(self): if OsUtils.isWin(): msysdir = CraftCore.standardDirs.msysDir() bash = CraftCore.cache.findApplication("bash", os.path.join(msysdir, "usr", "bin")) else: bash = CraftCore.cache.findApplication("bash") if not bash: CraftCore.log.critical("Failed to detect bash") return bash def execute(self, path, cmd, args="", **kwargs): # try to locate the command tmp = CraftCore.cache.findApplication(cmd) if tmp: cmd = tmp if CraftCore.compiler.isWindows: command = f"{self._findBash()} -c \"{self.toNativePath(cmd)} {args}\"" else: command = f"{self.toNativePath(cmd)} {args}" env = dict(os.environ) env.update(self.environment) env.update(kwargs.get("env", {})) return utils.system(command, cwd=path, env=env,**kwargs) def login(self): if CraftCore.compiler.isMSVC(): self.useMSVCCompatEnv = True return self.execute(os.curdir, self._findBash(), "-i", displayProgress=True) class Powershell(object): def __init__(self): - self.pwsh = CraftCore.cache.findApplication("pws") + self.pwsh = CraftCore.cache.findApplication("pwsh") if not self.pwsh: if platform.architecture()[0] == "32bit": self.pwsh = CraftCore.cache.findApplication("powershell", os.path.join(os.environ["WINDIR"], "sysnative", "WindowsPowerShell", "v1.0" )) if not self.pwsh: self.pwsh = CraftCore.cache.findApplication("powershell") if not self.pwsh: CraftCore.log.warning("Failed to detect powershell") def quote(self, s : str) -> str: return f"'{s}'" def execute(self, args :[str]) -> bool: return utils.system([self.pwsh, "-NoProfile", "-ExecutionPolicy", "ByPass", "-Command"] + args) def main(): shell = BashShell() shell.login() def testColor(): shell = BashShell() shell.execute(CraftCore.standardDirs.craftRoot(), os.path.join(CraftCore.standardDirs.craftBin(), "data", "ansi_color.sh")) if __name__ == '__main__': if len(sys.argv) > 1: if sys.argv[1] == "color": testColor() else: main()