diff --git a/bin/Packager/AppxPackager.py b/bin/Packager/AppxPackager.py index a9f3ef12a..112199927 100644 --- a/bin/Packager/AppxPackager.py +++ b/bin/Packager/AppxPackager.py @@ -1,170 +1,163 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import os import mimetypes from Utils import CraftHash from Packager.CollectionPackagerBase import * from Packager.PortablePackager import * from Blueprints.CraftVersion import CraftVersion class AppxPackager(CollectionPackagerBase): Extensions = f""" @{{file_types}} """ @InitGuard.init_once def __init__(self, whitelists=None, blacklists=None): CollectionPackagerBase.__init__(self, whitelists, blacklists) @staticmethod def _setupFileTypes(defines): if "mimetypes" in defines: defines.setdefault("file_types", set()) mimetypes.init() for t in defines["mimetypes"]: types = set(mimetypes.guess_all_extensions(t)) #remove reserved associations types -= {".bat", ".com", ".exe"} defines["file_types"] += types del defines["mimetypes"] if "file_types" in defines: defines["file_types"] = "\n".join([f"""{t}""" for t in set(defines["file_types"])]) defines.setdefault("extensions", AppxPackager.Extensions) else: defines.setdefault("file_types", "") defines.setdefault("extensions", "") - def _setDefaults(self, defines : dict) -> dict: - defines = dict(defines) + def setDefaults(self, defines : dict) -> dict: + defines = super().setDefaults(defines) if "version" not in defines: version = str(CraftVersion(self.version).strictVersion) # we require a version of the format 1.2.3.4 count = version.count(".") if count < 4: version = f"{version}{'.0' * (3-count)}" defines.setdefault("version", version) - - defines.setdefault("architecture", CraftCore.compiler.architecture) - defines.setdefault("company", "KDE e.V.") - defines.setdefault("display_name", self.subinfo.displayName) defines.setdefault("name", f"{defines['company']}{defines['display_name']}".replace(" ", "")) - defines.setdefault("craft_id", self.package.path.replace("/", ".")) - defines.setdefault("description", self.subinfo.description) - defines.setdefault("icon_png", os.path.join(CraftCore.standardDirs.craftBin(), "data", "icons", "craftyBENDER.png")) - defines.setdefault("icon_png_44", defines["icon_png"]) defines.setdefault("setupname", os.path.join(self.packageDestinationDir(), self.binaryArchiveName(fileType="appx", includeRevision=True))) + defines.setdefault("craft_id", self.package.path.replace("/", ".")) self._setupFileTypes(defines) # compat with nsis if "shortcuts" in self.defines: defines.setdefault("executable", self.defines["shortcuts"][0]["target"]) return defines def __prepareIcons(self, defines): utils.createDir(os.path.join(self.archiveDir(), "Assets")) defines["logo"] = os.path.join('Assets', os.path.basename(defines["icon_png"])) for propertyName, define, required in [ ("Square150x150Logo", "icon_png", True), ("Square44x44Logo", "icon_png_44", True), ("Wide310x150Logo", "icon_png_310x150", False), ("Square310x310Logo", "icon_png_310x310", False), ]: if define not in defines: if required: CraftCore.log.info(f"Please add defines[\"{define}]\"") return False else: defines[define] = "" continue icon = defines[define] defines[define] = f"{propertyName}=\"{os.path.join('Assets', os.path.basename(icon))}\"" names = glob.glob("{0}*{1}".format(*os.path.splitext(icon))) if not names: CraftCore.log.error(f"Failed to find {icon}") return False for n in names: if not utils.copyFile(n, os.path.join(self.archiveDir(), "Assets", os.path.basename(n))): return False return True def __createAppX(self, defines) -> bool: archive = defines["setupname"] if os.path.isfile(archive): utils.deleteFile(archive) return (utils.configureFile(os.path.join(os.path.dirname(__file__), "AppxManifest.xml"), os.path.join(self.archiveDir(), "AppxManifest.xml"), defines) and utils.system(["makeappx", "pack", "/d", self.archiveDir(), "/p", archive])) def __createSideloadAppX(self, defines) -> bool: def appendToPublisherString(publisher: [str], field: str, key: str) -> None: data = CraftCore.settings.get("CodeSigning", key, "") if data: publisher += [f"{field}={data}"] publisher = [] appendToPublisherString(publisher, "CN", "CommonName") appendToPublisherString(publisher, "O", "Organization") appendToPublisherString(publisher, "L", "Locality") appendToPublisherString(publisher, "S", "State") appendToPublisherString(publisher, "C", "Country") defines["publisher"] = ", ".join(publisher) setupName = "{0}-sideload{1}".format(*os.path.splitext(defines["setupname"])) defines["setupname"] = setupName return self.__createAppX(defines) and utils.sign([setupName]) def createPackage(self): - defines = self._setDefaults(self.defines) + defines = self.setDefaults(self.defines) if not "executable" in defines: CraftCore.log.error("Please add self.defines['shortcuts'] to the installer defines. e.g.\n" """self.defines["shortcuts"] = [{"name" : "Kate", "target":"bin/kate.exe", "description" : self.subinfo.description}]""") return False if not self.internalCreatePackage(): return False if not self.__prepareIcons(defines): return False publisher = CraftCore.settings.get("Packager", "AppxPublisherId", "") if publisher: defines.setdefault("publisher", publisher) if not self.__createAppX(defines): return False return self.__createSideloadAppX(defines) diff --git a/bin/Packager/CollectionPackagerBase.py b/bin/Packager/CollectionPackagerBase.py index 164c219db..4995143f8 100644 --- a/bin/Packager/CollectionPackagerBase.py +++ b/bin/Packager/CollectionPackagerBase.py @@ -1,277 +1,297 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # copyright (c) 2010-2011 Patrick Spendrin # copyright (c) 2010 Andre Heinecke (code taken from the kdepim-ce-package.py) # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import types import glob from Packager.PackagerBase import * from Blueprints.CraftDependencyPackage import DependencyType, CraftDependencyPackage from Blueprints.CraftPackageObject import * from Package.SourceOnlyPackageBase import * def toRegExp(fname, targetName) -> re: """ Read regular expressions from fname """ assert os.path.isabs(fname) if not os.path.isfile(fname): CraftCore.log.critical("%s not found at: %s" % (targetName.capitalize(), os.path.abspath(fname))) regex = [] with open(fname, "rt+") as f: for line in f: # Cleanup white spaces / line endings line = line.strip() if not line or line.startswith("#"): continue try: # accept forward and backward slashes line = line.replace("/", r"[/\\]") tmp = f"^{line}$" re.compile(tmp) # for debug regex.append(tmp) CraftCore.log.debug(f"{line} added to {targetName} as {tmp}") except re.error as e: raise Exception(f"{tmp} is not a valid regular expression: {e}") return re.compile(f"({'|'.join(regex)})", re.IGNORECASE) class PackagerLists(object): """ This class provides some staticmethods that can be used as pre defined black or whitelists """ @staticmethod def runtimeBlacklist(): return os.path.join(os.path.dirname(os.path.abspath(__file__)), "applications_blacklist.txt") @staticmethod def defaultWhitelist(): return [re.compile("^$")] @staticmethod def defaultBlacklist(): return [toRegExp(PackagerLists.runtimeBlacklist(), "blacklist")] class CollectionPackagerBase(PackagerBase): reMsvcDebugRt = re.compile(r"VCRUNTIME.*D\.DLL", re.IGNORECASE) @InitGuard.init_once def __init__(self, whitelists=None, blacklists=None): PackagerBase.__init__(self) if not whitelists: whitelists = [PackagerLists.defaultWhitelist] if not blacklists: blacklists = [PackagerLists.defaultBlacklist] if not self.whitelist_file: self.whitelist_file = whitelists if not self.blacklist_file: self.blacklist_file = blacklists self._whitelist = [] self._blacklist = [] self.scriptname = None self.__deployQtSdk = (OsUtils.isWin() and CraftCore.settings.getboolean("QtSDK", "Enabled", False) and CraftCore.settings.getboolean("QtSDK","PackageQtSDK",True)) self.__qtSdkDir = OsUtils.toNativePath(os.path.join(CraftCore.settings.get("QtSDK", "Path"), CraftCore.settings.get("QtSDK", "Version"), CraftCore.settings.get("QtSDK", "Compiler"))) if self.__deployQtSdk else None + def setDefaults(self, defines: {str:str}) -> {str:str}: + defines = dict(defines) + defines.setdefault("architecture", CraftCore.compiler.architecture) + defines.setdefault("company", "KDE") + defines.setdefault("productname", self.subinfo.displayName) + defines.setdefault("display_name", self.subinfo.displayName) + defines.setdefault("description", self.subinfo.description) + defines.setdefault("icon", os.path.join(CraftCore.standardDirs.craftBin(), "data", "icons", "craft.ico")) + defines.setdefault("icon_png", os.path.join(CraftCore.standardDirs.craftBin(), "data", "icons", "craftyBENDER.png")) + defines.setdefault("icon_png_44", defines["icon_png"]) + defines.setdefault("license", "") + defines.setdefault("version", self.sourceRevision() if self.subinfo.hasSvnTarget() else self.version) + defines.setdefault("website", + self.subinfo.webpage if self.subinfo.webpage else "https://community.kde.org/Craft") + + # mac + defines.setdefault("apppath", "") + defines.setdefault("appname", self.package.name.lower()) + return defines + @property def whitelist(self): if not self._whitelist: for entry in self.whitelist_file: CraftCore.log.debug("reading whitelist: %s" % entry) if callable(entry): for line in entry(): self._whitelist.append(line) else: self._whitelist.append(self.read_whitelist(entry)) return self._whitelist @property def blacklist(self): if not self._blacklist: for entry in self.blacklist_file: CraftCore.log.debug("reading blacklist: %s" % entry) if callable(entry): if entry == PackagerLists.runtimeBlacklist: CraftCore.log.warn("Compat mode for PackagerLists.runtimeBlacklist -- please just use self.blacklist_file.append(\"myblacklist.txt\") instead of self.blacklist_file = [...]") self._blacklist.append(self.read_blacklist(entry())) continue for line in entry(): self._blacklist.append(line) else: self._blacklist.append(self.read_blacklist(entry)) return self._blacklist def __imageDirPattern(self, package, buildTarget): """ return base directory name for package related image directory """ directory = "image" if package.subinfo.options.useBuildType == True: directory += '-' + package.buildType() directory += '-' + buildTarget return directory def __getImageDirectories(self): """ return the image directories where the files are stored """ imageDirs = [] depList = CraftDependencyPackage(self.package).getDependencies(depType=DependencyType.Runtime|DependencyType.Packaging, ignoredPackages=self.ignoredPackages) for x in depList: _package = x.instance if isinstance(_package, SourceOnlyPackageBase): CraftCore.log.debug(f"Ignoring package it is source only: {x}") continue imageDirs.append((x.instance.imageDir(), x.subinfo.options.package.disableStriping)) # this loop collects the files from all image directories CraftCore.log.debug(f"__getImageDirectories: package: {x}, version: {x.version}") if self.__deployQtSdk: imageDirs.append((self.__qtSdkDir, False)) return imageDirs def read_whitelist(self, fname : str) -> re: if not os.path.isabs(fname): fname = os.path.join(self.packageDir(), fname) """ Read regular expressions from fname """ try: return toRegExp(fname, "whitelist") except Exception as e: raise BlueprintException(str(e), self.package) def read_blacklist(self, fname : str) -> re: if not os.path.isabs(fname): fname = os.path.join(self.packageDir(), fname) """ Read regular expressions from fname """ try: return toRegExp(fname, "blacklist") except Exception as e: raise BlueprintException(str(e), self.package) def whitelisted(self, filename : os.DirEntry, root : str, whiteList : [re]=None) -> bool: """ return True if pathname is included in the pattern, and False if not """ if whiteList is None: whiteList = self.whitelist return self.blacklisted(filename, root=root, blackList=whiteList, message="whitelisted") def blacklisted(self, filename : os.DirEntry, root : str, blackList : [re]=None, message : str="blacklisted") -> bool: """ return False if file is not blacklisted, and True if it is blacklisted """ if blackList is None: blackList = self.blacklist CraftCore.log.debug(f"Start filtering: {message}") return utils.regexFileFilter(filename, root, blackList) def _filterQtBuildType(self, filename): if not self.__deployQtSdk: return True filename = OsUtils.toNativePath(filename) if self.__qtSdkDir not in filename: return True if utils.isBinary(filename): if not CraftCore.cache.findApplication("dependencies"): raise BlueprintException("Deploying a QtSdk depends on dev-util/dependencies", CraftPackageObject.get("dev-util/dependencies")) _, imports = CraftCore.cache.getCommandOutput("dependencies", f"-imports {filename}") rt = CollectionPackagerBase.reMsvcDebugRt.findall(imports) out = False if self.buildType() == "Debug": out = rt is not [] else: out = not rt if not out: CraftCore.log.debug(f"Skipp {filename} as it has the wrong build type: {rt}") return out return True def copyFiles(self, srcDir, destDir, dontStrip, symbolDest) -> bool: """ Copy the binaries for the Package from srcDir to the imageDir directory """ CraftCore.log.debug("Copying %s -> %s" % (srcDir, destDir)) doSign = CraftCore.compiler.isWindows and CraftCore.settings.getboolean("CodeSigning", "Enabled", False) for entry in utils.filterDirectoryContent(srcDir, self.whitelisted, self.blacklisted): if not self._filterQtBuildType(entry): continue entry_target = os.path.join(destDir, os.path.relpath(entry, srcDir)) if not utils.copyFile(entry, entry_target, linkOnly=False): return False if utils.isBinary(entry_target): if CraftCore.compiler.isGCCLike() and not dontStrip: self.strip(entry_target, symbolDest=symbolDest) if doSign: utils.sign([entry_target]) return True def internalCreatePackage(self, seperateSymbolFiles=False) -> bool: """ create a package """ archiveDir = self.archiveDir() CraftCore.log.debug("cleaning package dir: %s" % archiveDir) utils.cleanDirectory(archiveDir) if not seperateSymbolFiles: self.blacklist.append(re.compile(r".*\.pdb")) dbgDir = None else: dbgDir = f"{archiveDir}-dbg" utils.cleanDirectory(dbgDir) for directory, strip in self.__getImageDirectories(): if os.path.exists(directory): if not self.copyFiles(directory, archiveDir, strip, symbolDest=dbgDir): return False else: CraftCore.log.critical("image directory %s does not exist!" % directory) return False if self.subinfo.options.package.movePluginsToBin: # Qt expects plugins and qml files below bin, on the target sytsem binPath = os.path.join(archiveDir, "bin") for path in [os.path.join(archiveDir, "plugins"), os.path.join(archiveDir, "qml")]: if os.path.isdir(path): if not utils.mergeTree(path, binPath): return False if not self.preArchive(): return False if seperateSymbolFiles and CraftCore.compiler.isMSVC(): for f in glob.glob(f"{archiveDir}/**/*.pdb", recursive=True): dest = os.path.join(dbgDir, os.path.relpath(f, archiveDir)) utils.createDir(os.path.dirname(dest)) utils.moveFile(f, dest) return True def preArchive(self): return True diff --git a/bin/Packager/MacDMGPackager.py b/bin/Packager/MacDMGPackager.py index 11a37a320..64ece5614 100644 --- a/bin/Packager/MacDMGPackager.py +++ b/bin/Packager/MacDMGPackager.py @@ -1,383 +1,378 @@ from Packager.CollectionPackagerBase import * from Blueprints.CraftPackageObject import CraftPackageObject from Utils import CraftHash from pathlib import Path import contextlib import io import subprocess import stat import glob class MacDMGPackager( CollectionPackagerBase ): @InitGuard.init_once def __init__(self, whitelists=None, blacklists=None): CollectionPackagerBase.__init__(self, whitelists, blacklists) - def _setDefaults(self): - # TODO: Fix defaults - self.defines.setdefault("apppath", "") - self.defines.setdefault("appname", self.package.name.lower()) - def createPackage(self): """ create a package """ CraftCore.log.debug("packaging using the MacDMGPackager") packageSymbols = CraftCore.settings.getboolean("Packager", "PackageDebugSymbols", False) if not self.internalCreatePackage(seperateSymbolFiles=packageSymbols): return False - self._setDefaults() + defines = self.setDefaults(self.defines) archive = os.path.normpath(self.archiveDir()) - appPath = self.defines['apppath'] + appPath = defines['apppath'] if not appPath: - apps = glob.glob(os.path.join(archive, f"**/{self.defines['appname']}.app"), recursive=True) + apps = glob.glob(os.path.join(archive, f"**/{defines['appname']}.app"), recursive=True) if len(apps) != 1: - CraftCore.log.error(f"Failed to detect {self.defines['appname']}.app for {self}, please provide a correct self.defines['apppath'] or a relative path to the app as self.defines['apppath']") + CraftCore.log.error(f"Failed to detect {defines['appname']}.app for {self}, please provide a correct self.defines['apppath'] or a relative path to the app as self.defines['apppath']") return False appPath = apps[0] appPath = os.path.join(archive, appPath) appPath = os.path.normpath(appPath) CraftCore.log.info(f"Packaging {appPath}") targetLibdir = os.path.join(appPath, "Contents", "Frameworks") utils.createDir(targetLibdir) moveTargets = [ (os.path.join(archive, "lib", "plugins"), os.path.join(appPath, "Contents", "PlugIns")), (os.path.join(archive, "plugins"), os.path.join(appPath, "Contents", "PlugIns")), (os.path.join(archive, "lib"), targetLibdir), (os.path.join(archive, "share"), os.path.join(appPath, "Contents", "Resources"))] if not appPath.startswith(archive): moveTargets += [(os.path.join(archive, "bin"), os.path.join(appPath, "Contents", "MacOS"))] for src, dest in moveTargets: if os.path.exists(src): if not utils.mergeTree(src, dest): return False dylibbundler = MacDylibBundler(appPath) with utils.ScopedEnv({'DYLD_FALLBACK_LIBRARY_PATH': targetLibdir + ":" + os.path.join(CraftStandardDirs.craftRoot(), "lib")}): CraftCore.log.info("Bundling main binary dependencies...") - mainBinary = Path(appPath, "Contents", "MacOS", self.defines['appname']) + mainBinary = Path(appPath, "Contents", "MacOS", defines['appname']) if not dylibbundler.bundleLibraryDependencies(mainBinary): return False # Fix up the library dependencies of files in Contents/Frameworks/ CraftCore.log.info("Bundling library dependencies...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/Frameworks"): return False CraftCore.log.info("Bundling plugin dependencies...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/PlugIns"): return False if not utils.system(["macdeployqt", appPath, "-always-overwrite", "-verbose=1"]): return False # macdeployqt might just have added some explicitly blacklisted files blackList = Path(self.packageDir(), "mac_blacklist.txt") if blackList.exists(): pattern = [self.read_blacklist(str(blackList))] # use it as whitelist as we want only matches, ignore all others matches = utils.filterDirectoryContent(appPath, whitelist=lambda x, root: utils.regexFileFilter(x, root, pattern), blacklist=lambda x, root:True) for f in matches: CraftCore.log.info(f"Remove blacklisted file: {f}") utils.deleteFile(f) # macdeployqt adds some more plugins so we fix the plugins after calling macdeployqt dylibbundler.checkedLibs = set() # ensure we check all libs again (but # we should not need to make any changes) CraftCore.log.info("Fixing plugin dependencies after macdeployqt...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/PlugIns"): return False CraftCore.log.info("Fixing library dependencies after macdeployqt...") if not dylibbundler.fixupAndBundleLibsRecursively("Contents/Frameworks"): return False # Finally sanity check that we don't depend on absolute paths from the builder CraftCore.log.info("Checking for absolute library paths in package...") found_bad_dylib = False # Don't exit immeditately so that we log all the bad libraries before failing: if not dylibbundler.areLibraryDepsOkay(mainBinary): found_bad_dylib = True CraftCore.log.error("Found bad library dependency in main binary %s", mainBinary) if not dylibbundler.checkLibraryDepsRecursively("Contents/Frameworks"): CraftCore.log.error("Found bad library dependency in bundled libraries") found_bad_dylib = True if not dylibbundler.checkLibraryDepsRecursively("Contents/PlugIns"): CraftCore.log.error("Found bad library dependency in bundled plugins") found_bad_dylib = True if found_bad_dylib: CraftCore.log.error("Cannot not create .dmg since the .app contains a bad library depenency!") return False name = self.binaryArchiveName(fileType="", includeRevision=True) dmgDest = os.path.join(self.packageDestinationDir(), f"{name}.dmg") if os.path.exists(dmgDest): utils.deleteFile(dmgDest) - appName = self.defines['appname'] + ".app" + appName = defines['appname'] + ".app" if not utils.system(["create-dmg", "--volname", name, # Add a drop link to /Applications: "--icon", appName, "140", "150", "--app-drop-link", "350", "150", dmgDest, appPath]): return False CraftHash.createDigestFiles(dmgDest) return True class MacDylibBundler(object): """ Bundle all .dylib files that are not provided by the system with the .app """ def __init__(self, appPath: str): # Avoid processing the same file more than once self.checkedLibs = set() self.appPath = appPath def _addLibToAppImage(self, libPath: Path) -> bool: assert libPath.is_absolute(), libPath libBasename = libPath.name targetPath = Path(self.appPath, "Contents/Frameworks/", libBasename) if targetPath.exists() and targetPath in self.checkedLibs: return True # Handle symlinks (such as libgit2.27.dylib -> libgit2.0.27.4.dylib): if libPath.is_symlink(): linkTarget = os.readlink(str(libPath)) CraftCore.log.info("Library dependency %s is a symlink to '%s'", libPath, linkTarget) if os.path.isabs(linkTarget): CraftCore.log.error("%s: Cannot handle absolute symlinks: '%s'", libPath, linkTarget) return False if ".." in linkTarget: CraftCore.log.error("%s: Cannot handle symlinks containing '..': '%s'", libPath, linkTarget) return False if libPath.resolve().parent != libPath.parent.resolve(): CraftCore.log.error("%s: Cannot handle symlinks to other directories: '%s' (%s vs %s)", libPath, linkTarget, libPath.resolve().parent, libPath.parent.resolve()) return False # copy the symlink and add the real file: utils.copyFile(str(libPath), str(targetPath), linkOnly=False) CraftCore.log.info("Added symlink '%s' (%s) to bundle -> %s", libPath, os.readlink(str(targetPath)), targetPath) self.checkedLibs.add(targetPath) symlinkTarget = libPath.with_name(os.path.basename(linkTarget)) CraftCore.log.info("Processing symlink target '%s'", symlinkTarget) if not self._addLibToAppImage(symlinkTarget): self.checkedLibs.remove(targetPath) return False # If the symlink target was processed, the symlink itself is also fine return True if not libPath.exists(): CraftCore.log.error("Library dependency '%s' does not exist", libPath) return False CraftCore.log.debug("Handling library dependency '%s'", libPath) if not targetPath.exists(): utils.copyFile(str(libPath), str(targetPath), linkOnly=False) CraftCore.log.info("Added library dependency '%s' to bundle -> %s", libPath, targetPath) if not self._fixupLibraryId(targetPath): return False for path in utils.getLibraryDeps(str(targetPath)): # check there aren't any references to the original location: if path == str(libPath): CraftCore.log.error("%s: failed to fix reference to original location for '%s'", targetPath, path) return False if not self.bundleLibraryDependencies(targetPath): CraftCore.log.error("%s: UNKNOWN ERROR adding '%s' into bundle", targetPath, libPath) return False if not os.path.exists(targetPath): CraftCore.log.error("%s: Library dependency '%s' doesn't exist after copying... Symlink error?", targetPath, libPath) return False self.checkedLibs.add(targetPath) return True @staticmethod def _updateLibraryReference(fileToFix: Path, oldRef: str, newRef: str = None) -> bool: if newRef is None: newRef = "@executable_path/../Frameworks/" + os.path.basename(oldRef) with utils.makeWritable(fileToFix): if not utils.system(["install_name_tool", "-change", oldRef, newRef, str(fileToFix)], logCommand=False): CraftCore.log.error("%s: failed to update library dependency path from '%s' to '%s'", fileToFix, oldRef, newRef) return False return True @staticmethod def _getLibraryNameId(fileToFix: Path) -> str: libraryIdOutput = io.StringIO( subprocess.check_output(["otool", "-D", str(fileToFix)]).decode("utf-8").strip()) lines = libraryIdOutput.readlines() if len(lines) == 1: return "" # Should have exactly one line with the id now assert len(lines) == 2, lines return lines[1].strip() @classmethod def _fixupLibraryId(cls, fileToFix: Path): libraryId = cls._getLibraryNameId(fileToFix) if libraryId and os.path.isabs(libraryId): CraftCore.log.debug("Fixing library id name for %s", libraryId) with utils.makeWritable(fileToFix): if not utils.system(["install_name_tool", "-id", os.path.basename(libraryId), str(fileToFix)], logCommand=False): CraftCore.log.error("%s: failed to fix absolute library id name for", fileToFix) return False return True def bundleLibraryDependencies(self, fileToFix: Path) -> bool: assert not fileToFix.is_symlink(), fileToFix if fileToFix.stat().st_nlink > 1: CraftCore.log.error("More than one hard link to library %s found! " "This might modify another accidentally.", fileToFix) CraftCore.log.info("Fixing library dependencies for %s", fileToFix) if not self._fixupLibraryId(fileToFix): return False # Ensure we have the current library ID since we need to skip it in the otool -L output libraryId = self._getLibraryNameId(fileToFix) for path in utils.getLibraryDeps(str(fileToFix)): if path == libraryId: # The first line of the otool output is (usually?) the library itself: # $ otool -L PlugIns/printsupport/libcocoaprintersupport.dylib: # PlugIns/printsupport/libcocoaprintersupport.dylib: # libcocoaprintersupport.dylib (compatibility version 0.0.0, current version 0.0.0) # /System/Library/Frameworks/AppKit.framework/Versions/C/AppKit (compatibility version 45.0.0, current version 1561.40.112) # @rpath/QtPrintSupport.framework/Versions/5/QtPrintSupport (compatibility version 5.11.0, current version 5.11.1) # .... CraftCore.log.debug("%s: ignoring library name id %s in %s", fileToFix, path, os.path.relpath(str(fileToFix), self.appPath)) continue if path.startswith("@executable_path/"): continue # already fixed if path.startswith("@rpath/"): # CraftCore.log.info("%s: can't handle @rpath library dep of yet: '%s'", fileToFix, path) CraftCore.log.debug("%s: can't handle @rpath library dep of yet: '%s'", fileToFix, path) # TODO: run otool -l and verify that we pick the right file? elif path.startswith("/usr/lib/") or path.startswith("/System/Library/Frameworks/"): CraftCore.log.debug("%s: allowing dependency on system library '%s'", fileToFix, path) elif path.startswith("@loader_path/"): # TODO: we don't set @loader_path anymore so lets remove this after the next cache rebuild. 24.09.2018 if not self._updateLibraryReference(fileToFix, path): return False elif path.startswith("/"): if not path.startswith(CraftStandardDirs.craftRoot()): # TODO: should this be an error? CraftCore.log.warning("%s: reference to absolute library path outside craftroot: %s", fileToFix, path) # return False # file installed by craft -> bundle it into the .app if it doesn't exist yet if not self._addLibToAppImage(Path(path)): CraftCore.log.error("%s: Failed to add library dependency '%s' into bundle", fileToFix, path) return False if not self._updateLibraryReference(fileToFix, path): return False elif "/" not in path and path.startswith("lib"): # library reference without absolute path -> try to find the library # First check if it exists in Contents/Frameworks already guessedPath = Path(self.appPath, "Frameworks", path) if guessedPath.exists(): CraftCore.log.info("%s: relative library dependency is alreayd bundled: %s", fileToFix, guessedPath) else: guessedPath = Path(CraftStandardDirs.craftRoot(), "lib", path) if not guessedPath.exists(): CraftCore.log.error("%s: Could not find library dependency '%s' in craftroot", fileToFix, path) return False CraftCore.log.debug("%s: Found relative library reference %s in '%s'", fileToFix, path, guessedPath) if not self._addLibToAppImage(guessedPath): CraftCore.log.error("%s: Failed to add library dependency '%s' into bundle", fileToFix, guessedPath) return False if not self._updateLibraryReference(fileToFix, path): return False else: CraftCore.log.error("%s: don't know how to handle otool -L output: '%s'", fileToFix, path) return False return True def fixupAndBundleLibsRecursively(self, subdir: str): """Remove absolute references and budle all depedencies for all dylibs under :p subdir""" assert not subdir.startswith("/"), "Must be a relative path" for dirpath, dirs, files in os.walk(os.path.join(self.appPath, subdir)): for filename in files: fullpath = Path(dirpath, filename) if fullpath.is_symlink(): continue # No need to update symlinks since we will process the target eventually. if (filename.endswith(".so") or filename.endswith(".dylib") or ".so." in filename or (f"{fullpath.name}.framework" in str(fullpath) and utils.isBinary(str(fullpath)))): if not self.bundleLibraryDependencies(fullpath): CraftCore.log.info("Failed to bundle dependencies for '%s'", os.path.join(dirpath, filename)) return False return True def areLibraryDepsOkay(self, fullPath: Path): CraftCore.log.debug("Checking library dependencies of %s", fullPath) found_bad_lib = False libraryId = self._getLibraryNameId(fullPath) relativePath = os.path.relpath(str(fullPath), self.appPath) for dep in utils.getLibraryDeps(str(fullPath)): if dep == libraryId and not os.path.isabs(libraryId): continue # non-absolute library id is fine # @rpath and @executable_path is fine if dep.startswith("@rpath") or dep.startswith("@executable_path"): continue # Also allow /System/Library/Frameworks/ and /usr/lib: if dep.startswith("/usr/lib/") or dep.startswith("/System/Library/Frameworks/"): continue if dep.startswith(CraftStandardDirs.craftRoot()): CraftCore.log.error("ERROR: %s references absolute library path from craftroot: %s", relativePath, dep) elif dep.startswith("/"): CraftCore.log.error("ERROR: %s references absolute library path: %s", relativePath, dep) else: CraftCore.log.error("ERROR: %s has bad dependency: %s", relativePath, dep) found_bad_lib = True return not found_bad_lib def checkLibraryDepsRecursively(self, subdir: str): """Check that all absolute references and budle all depedencies for all dylibs under :p subdir""" assert not subdir.startswith("/"), "Must be a relative path" foundError = False for dirpath, dirs, files in os.walk(os.path.join(self.appPath, subdir)): for filename in files: fullpath = Path(dirpath, filename) if fullpath.is_symlink() and not fullpath.exists(): CraftCore.log.error("Found broken symlink '%s' (%s)", fullpath, os.readlink(str(fullpath))) foundError = True continue if filename.endswith(".so") or filename.endswith(".dylib") or ".so." in filename: if not self.areLibraryDepsOkay(fullpath): CraftCore.log.error("Found library dependency error in '%s'", fullpath) foundError = True return not foundError if __name__ == '__main__': print("Testing MacDMGPackager.py") defaultFile = CraftStandardDirs.craftRoot() + "/lib/libKF5TextEditor.5.dylib" sourceFile = defaultFile if len(sys.argv) else sys.argv[1] utils.system(["otool", "-L", sourceFile]) import tempfile with tempfile.TemporaryDirectory() as td: source = os.path.realpath(sourceFile) target = os.path.join(td, os.path.basename(source)) utils.copyFile(source, target, linkOnly=False) bundler = MacDylibBundler(td) bundler.bundleLibraryDependencies(Path(target)) print("Checked libs:", bundler.checkedLibs) utils.system(["find", td]) utils.system(["ls", "-laR", td]) if not bundler.areLibraryDepsOkay(Path(target)): print("Error") if not bundler.checkLibraryDepsRecursively("Contents/Frameworks"): print("Error 2") # utils.system(["find", td, "-type", "f", "-execdir", "otool", "-L", "{}", ";"]) diff --git a/bin/Packager/NullsoftInstallerPackager.py b/bin/Packager/NullsoftInstallerPackager.py index 2fd3c1b1f..afd712ce5 100644 --- a/bin/Packager/NullsoftInstallerPackager.py +++ b/bin/Packager/NullsoftInstallerPackager.py @@ -1,190 +1,183 @@ # -*- coding: utf-8 -*- # Copyright (c) 2010 Patrick Spendrin # Copyright (c) 2010 Andre Heinecke (code taken from the kdepim-ce-package.py) # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import os from Utils import CraftHash from Packager.CollectionPackagerBase import * from Packager.PortablePackager import * from Blueprints.CraftVersion import CraftVersion class NullsoftInstallerPackager(PortablePackager): """ Packager for Nullsoft scriptable install system This Packager generates a nsis installer (an executable which contains all files) from the image directories of craft. This way you can be sure to have a clean installer. In your package, you can add regexp whitelists and blacklists (see example files for the fileformat). The files for both white- and blacklists, must be given already in the constructor. You can override the .nsi default script and you will get the following defines given into the nsis generator via commandline if you do not override the attributes of the same name in the dictionary self.defines: setupname: PACKAGENAME-setup-BUILDTARGET.exe PACKAGENAME is the name of the package srcdir: is set to the image directory, where all files from the image directories of all dependencies are gathered. You shouldn't normally have to set this. company: sets the company name used for the registry key of the installer. Default value is "KDE". productname: contains the capitalized PACKAGENAME and the buildTarget of the current package executable: executable is defined empty by default, but it is used to add a link into the start menu. You can add your own defines into self.defines as well. """ @InitGuard.init_once def __init__(self, whitelists=None, blacklists=None): PortablePackager.__init__(self, whitelists, blacklists) self.nsisExe = None self._isInstalled = False - def _setDefaults(self, defines): - defines = dict(defines) - defines.setdefault("architecture", CraftCore.compiler.architecture) - defines.setdefault("company", "KDE") + def setDefaults(self, defines) -> {}: + defines = super().setDefaults(defines) defines.setdefault("defaultinstdir", "$PROGRAMFILES64" if CraftCore.compiler.isX64() else "$PROGRAMFILES") defines.setdefault("multiuser_use_programfiles64", "!define MULTIUSER_USE_PROGRAMFILES64" if CraftCore.compiler.isX64() else "") - defines.setdefault("icon", os.path.join(CraftCore.standardDirs.craftBin(), "data", "icons", "craft.ico")) - defines.setdefault("license", "") - defines.setdefault("productname", self.subinfo.displayName) defines.setdefault("setupname", self.binaryArchiveName(fileType="exe", includeRevision=True)) defines.setdefault("srcdir", self.archiveDir())# deprecated - defines.setdefault("version", self.sourceRevision() if self.subinfo.hasSvnTarget() else self.version) - defines.setdefault("website", self.subinfo.webpage if self.subinfo.webpage else "https://community.kde.org/Craft") defines.setdefault("registy_hook", "") defines.setdefault("sections", "") defines.setdefault("sections_page", "") defines.setdefault("preInstallHook", "") if not self.scriptname: self.scriptname = os.path.join(os.path.dirname(__file__), "Nsis", "NullsoftInstaller.nsi") return defines def isNsisInstalled(self): if not self._isInstalled: self._isInstalled = self.__isInstalled() if not self._isInstalled: CraftCore.log.critical("Craft requires Nsis to create a package, please install Nsis\n" "\t'craft nsis'") return False return True def __isInstalled(self): """ check if nsis (Nullsoft scriptable install system) is installed somewhere """ self.nsisExe = CraftCore.cache.findApplication("makensis") if not self.nsisExe: return False return CraftCore.cache.getVersion(self.nsisExe, versionCommand="/VERSION") >= CraftVersion("3.03") def _createShortcut(self, name, target, icon="", parameter="", description="") -> str: return f"""CreateShortCut "$SMPROGRAMS\$StartMenuFolder\\{name}.lnk" "$INSTDIR\\{OsUtils.toNativePath(target)}" "{parameter}" "{icon}" 0 SW_SHOWNORMAL "" "{description}"\n""" def folderSize(self, path): total = 0 for entry in os.scandir(path): if entry.is_file(): total += entry.stat().st_size elif entry.is_dir(): total += self.folderSize(entry.path) return total def generateNSISInstaller(self): """ runs makensis to generate the installer itself """ - defines = self._setDefaults(self.defines) + defines = self.setDefaults(self.defines) defines["dataPath"] = self.setupName defines["dataName"] = os.path.basename(self.setupName) defines["7za"] = CraftCore.cache.findApplication("7za") if CraftCore.compiler.isX64() else CraftCore.cache.findApplication("7za_32") # provide the actual installation size in kb, ignore the 7z size as it gets removed after the install defines["installSize"] = str(int((self.folderSize(self.archiveDir()) - os.path.getsize(self.setupName)) / 1000)) defines["installerIcon"] = f"""!define MUI_ICON "{defines["icon"]}" """ defines["iconname"] = os.path.basename(defines["icon"]) if not defines["license"] == "": defines["license"] = f"""!insertmacro MUI_PAGE_LICENSE "{defines["license"]}" """ shortcuts = [] if "executable" in defines: shortcuts.append(self._createShortcut(defines["productname"], defines["executable"])) del defines["executable"] for short in self.shortcuts: shortcuts.append(self._createShortcut(**short)) defines["shortcuts"] = "".join(shortcuts) if defines.get("sections", None): defines["sections_page"] = "!insertmacro MUI_PAGE_COMPONENTS" # make absolute path for output file if not os.path.isabs(defines["setupname"]): dstpath = self.packageDestinationDir() defines["setupname"] = os.path.join(dstpath, defines["setupname"]) self.setupName = defines["setupname"] CraftCore.debug.new_line() CraftCore.log.debug(f"generating installer {self.setupName}") verboseString = "/V4" if CraftCore.debug.verbose() > 0 else "/V3" defines.setdefault("nsis_include", f"!addincludedir {os.path.dirname(self.scriptname)}") defines["nsis_include_internal"] = f"!addincludedir {os.path.join(os.path.dirname(__file__), 'Nsis')}" cmdDefines = [] configuredScrip = os.path.join(self.workDir(), f"{self.package.name}.nsi") if not utils.configureFile(self.scriptname, configuredScrip, defines): configuredScrip = self.scriptname # this script uses the old behaviour, using defines for key, value in defines.items(): if value is not None: cmdDefines.append(f"/D{key}={value}") if not utils.systemWithoutShell([self.nsisExe, verboseString] + cmdDefines + [configuredScrip], cwd=os.path.abspath(self.packageDir())): CraftCore.log.critical("Error in makensis execution") return False return utils.sign([self.setupName]) def createPackage(self): """ create a package """ if not self.isNsisInstalled(): return False CraftCore.log.debug("packaging using the NullsoftInstallerPackager") if not super().createPackage(): return False if not self.generateNSISInstaller(): return False destDir, archiveName = os.path.split(self.setupName) self._generateManifest(destDir, archiveName) CraftHash.createDigestFiles(self.setupName) return True