diff --git a/CraftSettings.ini.template b/CraftSettings.ini.template index bf93dfa6a..6b57c9b2f 100644 --- a/CraftSettings.ini.template +++ b/CraftSettings.ini.template @@ -1,226 +1,232 @@ ## You can use cmake like variables for values in the same section ${Variable} ## or for variables from a different section ${Section:Variable}. ## See Paths/${DOWNLOADDIR} [General] ## Here you set the ABI to be used. ## #platform-#abi-#compiler ## Valid combinations are: ## windows-msvc[2015, 2017]_[32, 64]-[cl, clang] ## windows-mingw_[32, 64]-[gcc, clang] ## linux-[32, 64]-[gcc, clang] ## macos-[32, 64]-clang ## freebsd-[32, 64]-clang ABI = windows-msvc2015_64-cl ## This option should be set to False if you use the msvc 201X Express Edition 64bit compiler ## in all other cases, simply keep this option commented out #Native=False ## This option specifies which MSVC toolset to use. Leave this commented out for MSVC to pick ## the newest available toolset. ## This is an expert option, do not touch it unless you know exactly what it means. #MSVCToolset=14.16 +## Set the minimum supported MacOs version, this is also limited by Qt +## 10.13 https://github.com/qt/qtbase/blob/5.14/mkspecs/common/macx.conf#L8 +## 10.12 https://github.com/qt/qtbase/blob/5.13/mkspecs/common/macx.conf#L8 +## 10.12 https://github.com/qt/qtbase/blob/5.12/mkspecs/common/macx.conf#L8 +#MacDeploymentTarget = 10.13 + ## This option can be used to enable a notification backend. ## As soon as the buildprocess of a project has finished a notification will be displayed. ## Possible Backends: ## Snore: https://commits.kde.org/snorenotify. Snore supports multiple backends. ## Just 'craft snorenotify' ## To configure snorenotify for craft call 'snoresettings --appName snoresend' ## SnoreToast: simple command line util for Windows notifications Notify = SnoreToast ## Speed up the merging of packages by using hard links UseHardlinks = True [Variables] ## Values here are usually set by craft and can be used for dynamic values ## To override the variables, uncomment them ## The working directory of Craft #CraftRoot = ## The directory of the Craft clone #CraftDir = [Paths] ## This is the location of your python installation. ## This value must be set. Python = C:\PROGRA~1\Python36 ## Some applications may need python 2.7 #Python27 = C:\python27 ## Craft ist able to fetch and install Msys itself, but if you prefer to use ## your own installation specify it here #Msys = C:\msys ## Here you change the download directory. ## If you want, so you can share the same download directory between ## mingw and msvc. ## The default value is craft/../download #DownloadDir = C:\kde\download ## This option defines the location for git checkouts. ## The default value is craft/../download/git #KDEGitDir = ${DOWNLOADDIR}\git ## This option defines the location for svn checkouts. ## The default value is craft/../download/svn #KDESVNDir = ${DOWNLOADDIR}\svn ## This option defines the location where the ccache files are stored. ## The default location is KDEROOT/build/ccache #CCACHE_DIR = C:\CCACHE\kf5 [Compile] ## the buildtype of this installation ## Possible Values: ## Release ## RelWithDebInfo ## Debug ## MinSizeRel BuildType = RelWithDebInfo ## Whether to use ninja (default: False) UseNinja = True ## Whether to use ccache (only avalible with mingw compiler) #UseCCache = True ## This option can be used to override the default make program ## change the value to the path of the executable you want to use instead. MakeProgram = jom ## Number of build jobs ## The default is number of cores or determined by the build tool #Jobs = 2 [CMake] ## Fetch the translations for KDE projects when build from git KDE_L10N_AUTO_TRANSLATIONS = OFF [ShortPath] ## The directory where the junctions are created. #JunctionDir= [Blueprints] ## The location where the default blueprints are stored ## This is especially useful when using multiple Craft setups #BlueprintRoot = ${Variables:CraftRoot}/etc/blueprints/locations ## The locations of the recipes ## You can specify additional external locations in ; separated list #Locations = C:\blueprints # Customer settings Settings = ${Variables:CraftRoot}/etc/BlueprintSettings.ini [BlueprintVersions] ## Allow to automatically update certain recipes once a day. EnableDailyUpdates = True [Packager] ## The location where generated installers are placed #Destination = ${Variables:CraftRoot}/tmp ## The archive type for packages. ## Possible values are: zip, 7z ## Todo: rename #7ZipArchiveType = 7z # id assigned to you by the Windows Store #AppxPublisherId = CN=98B52D9A-DF7C-493E-BADC-37004A92EFC8 ## If set this will override the default package type. ## Possible values are: ### SevenZipPackager: An image of the files installed by the package ### MSIFragmentPackager ### NullsoftInstallerPackager: A nsis based installer including all dependencies ### CreateArchivePackager: An image including all dependencies #PackageType = SevenZipPackager ## Package the Source files too. PackageSrc = False ## Whether to package debug symbols ### by default debug symbols are stripped/removed from the package ### If PackageDebugSymbols is set to True, a separate archive with the symbols is created. PackageDebugSymbols = True ## A url to a Craft cache repository ## Sets a custom repository for the binary cache #RepositoryUrl = https://files.kde.org/craft/master/ ## Enable to fatch packages from a Craft cache repository ## See --use-cache and --no-cache in the Craft help. UseCache = True [CraftDebug] ## If you want to have verbose output, uncomment the following option ## and set it to positive integer for verbose output and to 0 ## (or disable it) for normal output. Currently the highest verbosity level ## is 3 (equal to 'craft -v -v -v'). level -1 equals 'craft -q' ## Default is Verbose = 0 #Verbose = 1 # Log environment, prints the current state of the environment before an application is run LogEnvironment = True PrintPutEnv = False ## Prints time spend on various craft tasks MeasureTime = False ## Dump internal state of craftSettings to kdesettings.ini.dump #DumpSettings = True ## Print function Deprecation messages LogDeprecated = True [Environment] ## All values defined here will be populated to the environment #GIT_COMMITTER_EMAIL = foo@bar.com ## Set the ssh client for git and svn. #GIT_SSH = plink #SVN_SSH = plink [QtSDK] ## For advanced users only ## Whether to use prebuild Qt binaries. Enabled = False ## The path to the Qt sdk. Path = D:\Qt ## The version of Qt. Version = 5.11.1 ## The compiler version, if you are not sure what to use, have a look into the directory set in QtSDK/Path. ## The compiler must be of the same type as General/KDECOMPILER. ## If you are using mingw please make sure you have installed the mingw using the Qt installer. Compiler = mingw482_32 [ContinuousIntegration] ## Changes the verbosity of some sub processes. ## Installed versions must match. ## Don't write to stderr Enabled = False ## Delete the build folder after the install ClearBuildFolder = False [CodeSigning] Enabled = False SignCache = ${CodeSigning:Enabled} #Certificate = C:\CraftRoot\mycert.pfx CommonName = K Desktop Environment e.V. Organization = K Desktop Environment e.V. Locality = Berlin Country = DE State = [Version] ConfigVersion = 6 diff --git a/bin/BuildSystem/CMakeBuildSystem.py b/bin/BuildSystem/CMakeBuildSystem.py index 55d5f946c..82209f902 100644 --- a/bin/BuildSystem/CMakeBuildSystem.py +++ b/bin/BuildSystem/CMakeBuildSystem.py @@ -1,133 +1,130 @@ # # copyright (c) 2009 Ralf Habacker # """@package provides cmake build system""" from BuildSystem.BuildSystemBase import * from CraftOS.osutils import OsUtils from CraftStandardDirs import CraftStandardDirs from Utils.PostInstallRoutines import * import os class CMakeBuildSystem(BuildSystemBase): """ cmake build support """ def __init__(self): """constructor. configureOptions are added to the configure command line and makeOptions are added to the make command line""" BuildSystemBase.__init__(self, "cmake") self.supportsNinja = True def __makeFileGenerator(self): """return cmake related make file generator""" if self.makeProgram == "ninja": return "Ninja" if OsUtils.isWin(): if CraftCore.compiler.isMSVC() and not CraftCore.compiler.isIntel(): return "NMake Makefiles" if CraftCore.compiler.isMinGW(): return "MinGW Makefiles" elif OsUtils.isUnix(): return "Unix Makefiles" else: CraftCore.log.critical(f"unknown {CraftCore.compiler} compiler") def configureOptions(self, defines=""): """returns default configure options""" options = "-DBUILD_TESTING={testing} ".format(testing="ON" if self.buildTests else "OFF") options += BuildSystemBase.configureOptions(self) craftRoot = OsUtils.toUnixPath(CraftCore.standardDirs.craftRoot()) options += f" -DCMAKE_INSTALL_PREFIX=\"{craftRoot}\"" options += f" -DCMAKE_PREFIX_PATH=\"{craftRoot}\"" if self.buildType() is not None: options += " -DCMAKE_BUILD_TYPE=%s" % self.buildType() #if CraftCore.compiler.isGCC() and not CraftCore.compiler.isNative(): # options += " -DCMAKE_TOOLCHAIN_FILE=%s" % os.path.join(CraftStandardDirs.craftRoot(), "craft", "bin", "toolchains", "Toolchain-cross-mingw32-linux-%s.cmake" % CraftCore.compiler.architecture) if CraftCore.settings.getboolean("CMake", "KDE_L10N_AUTO_TRANSLATIONS", False): options += " -DKDE_L10N_AUTO_TRANSLATIONS=ON" if OsUtils.isWin(): # people use InstallRequiredSystemLibraries.cmake wrong and unconditionally install the # msvc crt... options += " -DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=ON" if OsUtils.isMac(): options += f" -DKDE_INSTALL_BUNDLEDIR=\"{OsUtils.toUnixPath(CraftCore.standardDirs.craftRoot())}/Applications/KDE\" -DAPPLE_SUPPRESS_X11_WARNING=ON" - if CraftCore.compiler.macUseSDK: - # Ensure that we don't depend on SDK features only present on newer systems - options += " -DCMAKE_OSX_DEPLOYMENT_TARGET=" + CraftCore.compiler.macOSDeploymentTarget if CraftCore.compiler.isWindows or CraftCore.compiler.isMacOS: options += " -DKDE_INSTALL_USE_QT_SYS_PATHS=ON" if self.subinfo.options.buildTools: options += " " + self.subinfo.options.configure.toolsDefine + " " if self.subinfo.options.buildStatic and self.subinfo.options.configure.staticArgs: options += " " + self.subinfo.options.configure.staticArgs + " " if CraftCore.compiler.isIntel(): # this is needed because otherwise it'll detect the MSVC environment options += " -DCMAKE_CXX_COMPILER=\"%s\" " % os.path.join(os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"), "icl.exe").replace("\\", "/") options += " -DCMAKE_C_COMPILER=\"%s\" " % os.path.join(os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"), "icl.exe").replace("\\", "/") options += " -DCMAKE_LINKER=\"%s\" " % os.path.join(os.getenv("BIN_ROOT"), os.getenv("ARCH_PATH"), "xilink.exe").replace("\\", "/") options += " \"%s\"" % self.configureSourceDir() return options def configure(self, defines=""): """implements configure step for cmake projects""" self.enterBuildDir() command = r"""cmake -G "%s" %s""" % (self.__makeFileGenerator(), self.configureOptions(defines)) CraftCore.debug.step(command) return utils.system(command) def make(self): """implements the make step for cmake projects""" self.enterBuildDir() command = ' '.join([self.makeProgram, self.makeOptions(self.subinfo.options.make.args)]) return utils.system(command) def install(self): """install the target""" if not BuildSystemBase.install(self): return False self.enterBuildDir() with utils.ScopedEnv({"DESTDIR" : self.installDir()}): command = [self.makeProgram, self.makeOptions(self.subinfo.options.install.args)] return (utils.system(" ".join(command)) and self._fixInstallPrefix()) def unittest(self): """running cmake based unittests""" self.enterBuildDir() command = ["ctest", "--output-on-failure", "--timeout", "300"] if CraftCore.debug.verbose() == 1: command += ["-V"] elif CraftCore.debug.verbose() > 1: command += ["-VV"] return utils.system(command) def ccacheOptions(self): out = f" -DCMAKE_CXX_COMPILER=ccache -DCMAKE_CXX_COMPILER_ARG1={os.environ['CXX']}" out += f" -DCMAKE_C_COMPILER=ccache -DCMAKE_C_COMPILER_ARG1={os.environ['CC']} " return out def internalPostQmerge(self): if not super().internalPostQmerge(): return False return PostInstallRoutines.updateSharedMimeInfo(self) diff --git a/bin/CraftCompiler.py b/bin/CraftCompiler.py index 46ac4b887..3d556666f 100644 --- a/bin/CraftCompiler.py +++ b/bin/CraftCompiler.py @@ -1,330 +1,315 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import re from enum import unique, IntFlag import utils from CraftConfig import * from CraftCore import CraftCore from CraftDebug import deprecated class CraftCompiler(object): class Platforms(IntFlag): NoPlatform = 0 Windows = 0x1 << 0 Linux = 0x1 << 1 MacOS = 0x1 << 2 FreeBSD = 0x1 << 3 Unix = Linux | MacOS | FreeBSD All = ~0 # define inverted values to allow usage in info.ini NotLinux = ~Linux NotMacOS = ~MacOS NotFreeBSD = ~FreeBSD NotWindows = Unix NotUnix = ~Unix @classmethod def fromString(cls, name): if not hasattr(cls, "__sting_map"): cls.__sting_map = dict([(k.lower(), v) for k, v in cls.__members__.items()]) return cls.__sting_map[name.lower()] @unique class Compiler(IntFlag): NoCompiler = 0 CL = 0x1 << 0 GCC = 0x1 << 1 CLANG = 0x1 << 2 GCCLike = CLANG | GCC All = ~0 @classmethod def fromString(cls, name): if not hasattr(cls, "__sting_map"): cls.__sting_map = dict([(k.lower(), v) for k, v in cls.__members__.items()]) return cls.__sting_map[name.lower()] def __init__(self): compiler = CraftCore.settings.get("General", "KDECOMPILER", "") if compiler != "": arch = "32" if CraftCore.settings.get("General", "Architecture") == "x86" else "64" if compiler.startswith("msvc"): split = ["windows", f"{compiler}_{arch}", "cl"] elif compiler.startswith("mingw"): split = ["windows", f"mingw_{arch}", "gcc"] elif compiler.startswith("linux"): split = ["linux", "gcc"] elif compiler.startswith("mac"): split = ["macos", "clang"] if not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): print(f"Your using the old compiler setting\n" f"\t[General]\n" f"\tKDECOMPILER={compiler}\n" f"please update your settings to\n" f"\t[General]\n" f"\tABI=" + "-".join(split), file=sys.stderr) else: split = CraftCore.settings.get("General", "ABI").split("-") if len(split) != 3: raise Exception("Invalid compiler: " + CraftCore.settings.get("General", "ABI")) platform, self._abi, compiler = split self._compiler = CraftCompiler.Compiler.fromString(compiler) self._platform = CraftCompiler.Platforms.fromString(platform) self._architecture = "x86" if self._abi.endswith("32") else "x64" self._MSVCToolset = None if self.isMSVC(): self._MSVCToolset = CraftCore.settings.get("General", "MSVCToolset", "") def __str__(self): return "-".join(self.signature) @property def signature(self): return self.platform.name.lower(), self.abi, self.compiler.name.lower() @property def platform(self) -> Platforms: return self._platform @property def abi(self): return self._abi @property def compiler(self) -> Compiler: return self._compiler @property def architecture(self): return self._architecture @property def msvcToolset(self): return self._MSVCToolset @property def gnuArchitecture(self): return "x86" if self.isX86() else "x86_64" @property def bits(self): return "64" if self.isX64() else "32" def _getGCCTarget(self): _, result = CraftCore.cache.getCommandOutput("gcc", "-dumpmachine") if result: result = result.strip() CraftCore.log.debug(f"GCC Target Processor: {result}") else: # if no mingw is installed return mingw-w32 it is part of base if self.isX64(): result = "x86_64-w64-mingw32" else: result = "i686-w64-mingw32" return result - @property - def macOSDeploymentTarget(self) -> str: - assert self.macUseSDK - return "10.11" - - @property - def macUseSDK(self) -> bool: - assert self.isMacOS - """Whether to compile against the macOS SDK instead of the libraries in /usr (off by default)""" - return CraftCore.settings.getboolean("MacSDK", "Enabled", "False") - @property def isWindows(self) -> bool: return self.platform == CraftCompiler.Platforms.Windows @property def isMacOS(self) -> bool: return self.platform == CraftCompiler.Platforms.MacOS @property def isLinux(self) -> bool: return self.platform == CraftCompiler.Platforms.Linux @property def isFreeBSD(self) -> bool: return self.platform == CraftCompiler.Platforms.FreeBSD @property def isUnix(self) -> bool: return bool(self.platform & CraftCompiler.Platforms.Unix) @property def executableSuffix(self): return ".exe" if self.isWindows else "" def isNative(self): return CraftCore.settings.getboolean("General", "Native", True) def isX64(self): return self.architecture == "x64" def isX86(self): return self.architecture == "x86" def isGCC(self) -> bool: return self.compiler == CraftCompiler.Compiler.GCC def isClang(self) -> bool: return self.compiler == CraftCompiler.Compiler.CLANG def isGCCLike(self) -> bool: return bool(self.compiler & CraftCompiler.Compiler.GCCLike) def isCl(self) -> bool: return self.compiler == CraftCompiler.Compiler.CL def isMinGW(self): return self.abi.startswith("mingw") def isMinGW_W32(self): return self.isMinGW() and self.isX86() def isMinGW_W64(self): return self.isMinGW() and self.isX64() def isMSVC(self): return self.abi.startswith("msvc") def isMSVC2010(self): return self.abi.startswith("msvc2010") def isMSVC2012(self): return self.abi.startswith("msvc2012") def isMSVC2013(self): return self.abi.startswith("msvc2013") def isMSVC2015(self): return self.abi.startswith("msvc2015") def isMSVC2017(self): return self.abi.startswith("msvc2017") def isMSVC2019(self): return self.abi.startswith("msvc2019") def isIntel(self): return self.compiler == "intel" @deprecated("CraftCore.compiler") def getCompilerName(self): return str(CraftCore.compiler) @deprecated("CraftCore.compiler") def getSimpleCompilerName(self): return str(CraftCore.compiler) def getGCCLikeVersion(self, compilerExecutable): _, result = CraftCore.cache.getCommandOutput(compilerExecutable, "--version") if result: result = re.findall("\d+\.\d+\.?\d*", result)[0] CraftCore.log.debug("{0} Version: {1}".format(compilerExecutable, result)) return result or "0" def getVersion(self): if self.isGCCLike(): return self.getGCCLikeVersion(self.compiler.name) elif self.isMSVC(): return self.getInternalVersion() else: return None def getVersionWithName(self): if self.isGCCLike(): return f"{self.getCompilerName()} {self.getVersion()}" elif self.isIntel(): return os.getenv("PRODUCT_NAME_FULL") elif self.isMSVC(): return f"Microsoft Visual Studio {self.getVersion()}" else: return None def getShortName(self): if not self.isMSVC(): return self.getCompilerName() return f"vc{self.getInternalVersion()}" def getInternalVersion(self): if not self.isMSVC(): return self.getVersion() versions = { "msvc2010": 10, "msvc2012": 11, "msvc2013": 12, "msvc2015": 14, "msvc2017": 15, "msvc2019": 16 } c = self.abi.split("_")[0] if c not in versions: CraftCore.log.critical(f"Unknown MSVC Compiler {self.abi}") return versions[c] def getMsvcPlatformToolset(self): versions = { "msvc2010": 100, "msvc2012": 110, "msvc2013": 120, "msvc2015": 140, "msvc2017": 141, "msvc2019": 142 } c = self.abi.split("_")[0] if c not in versions: CraftCore.log.critical(f"Unknown MSVC Compiler {self.abi}") return versions[c] if __name__ == '__main__': print("Testing Compiler.py") print(f"Configured compiler (ABI): {CraftCore.compiler}") print("Version: %s" % CraftCore.compiler.getVersionWithName()) print("Compiler Name: %s" % CraftCore.compiler.getCompilerName()) print("Native compiler: %s" % ("No", "Yes")[CraftCore.compiler.isNative()]) if CraftCore.compiler.isGCCLike(): print("Compiler Version: %s" % CraftCore.compiler.getGCCLikeVersion(CraftCore.compiler.compiler.name)) if CraftCore.compiler.isGCC(): - print("Compiler Target: %s" % CraftCore.compiler._getGCCTarget()) - if CraftCore.compiler.isMacOS: - print("Using SDK for macOS:", CraftCore.compiler.macUseSDK) - if CraftCore.compiler.macUseSDK: - print("macOS SDK deployment target:", CraftCore.compiler.macOSDeploymentTarget) + print("Compiler Target: %s" % CraftCore.compiler._getGCCTarget()) \ No newline at end of file diff --git a/bin/CraftSetupHelper.py b/bin/CraftSetupHelper.py index 782f062e0..71c42abc6 100644 --- a/bin/CraftSetupHelper.py +++ b/bin/CraftSetupHelper.py @@ -1,454 +1,455 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import argparse import collections import os import platform import shutil import subprocess import sys from CraftCore import CraftCore from CraftOS.osutils import OsUtils from Utils.CaseInsensitiveDict import CaseInsensitiveDict # The minimum python version for craft please edit here # if you add code that changes this requirement MIN_PY_VERSION = (3, 6, 0) def log(msg, critical=False): if critical or not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): CraftCore.debug.print(msg, sys.stderr) else: CraftCore.log.debug(msg) if critical: exit(1) if sys.version_info[0:3] < MIN_PY_VERSION: log("Error: Python too old!\n" "Craft needs at least Python Version %s.%s.%s\n" "Please install it and adapt your CraftSettings.ini" % MIN_PY_VERSION, critical=True) if not platform.machine().endswith("64"): log(f"Craft requires a 64bit operating system. Your are using: {platform.machine()}", critical=True) class SetupHelper(object): CraftVersion = "master" NeedsSetup = "KDEROOT" not in os.environ def __init__(self, args=None): self.args = args if CraftCore.settings.getboolean("General", "AllowAnsiColor", False): OsUtils.enableAnsiColors() if SetupHelper.NeedsSetup: SetupHelper.NeedsSetup = False self.checkForEvilApplication() self.setupEnvironment() @staticmethod def _getOutput(command, shell=False): CraftCore.log.debug(f"SetupHelper._getOutput: {command}") p = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, universal_newlines=True, errors="backslashreplace") out = p.stdout.strip() CraftCore.log.debug(f"SetupHelper._getOutput: return {p.returncode} {out}") return p.returncode, out def run(self): parser = argparse.ArgumentParser() parser.add_argument("--get", action="store_true") parser.add_argument("--print-banner", action="store_true") parser.add_argument("--getenv", action="store_true") parser.add_argument("--setup", action="store_true") parser.add_argument("rest", nargs=argparse.REMAINDER) args = parser.parse_args() if args.get: default = "" if len(args.rest) == 3: default = args.rest[2] CraftCore.log.info(CraftCore.settings.get(args.rest[0], args.rest[1], default)) elif args.print_banner: self.printBanner() elif args.getenv: self.printEnv() elif args.setup: self.printEnv() self.printBanner() def checkForEvilApplication(self): blackList = [] if OsUtils.isWin(): blackList += ["sh", "gcc", "g++", "cpp"] for app in blackList: location = shutil.which(app) if location: location = os.path.dirname(location) if not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): log( f"Found \"{app}\" in your PATH: \"{location}\"\n" f"This application is known to cause problems with your configuration of Craft.\n" f"Please remove it from PATH or manually set a value for PATH in your CraftSettings.ini:\n" f"\n" f"[Environment]\n" f"PATH=" f"\n") else: path = collections.OrderedDict.fromkeys(os.environ["Path"].split(os.path.pathsep)) del path[location] self.addEnvVar("Path", os.path.pathsep.join(path)) @staticmethod def printBanner(): def printRow(name, value): log(f"{name:20}: {value}") printRow("Craft", CraftCore.standardDirs.craftRoot()) printRow("Version", SetupHelper.CraftVersion) printRow("ABI", CraftCore.compiler) printRow("Download directory", CraftCore.standardDirs.downloadDir()) def addEnvVar(self, key, val): os.environ[key] = val def prependEnvVar(self, key : str, var : str, sep : str=os.path.pathsep) -> None: if not type(var) == list: var = [var] if key in os.environ: env = var + os.environ[key].split(sep) var = list(collections.OrderedDict.fromkeys(env)) val = sep.join(var) CraftCore.log.debug(f"Setting {key}={val}") os.environ[key] = val @staticmethod def stringToEnv(string : str): env = CaseInsensitiveDict(os.environ) for line in string.strip().split("\n"): kv = line.strip().split("=", 1) if len(kv) != 2: log(f"Failed to parse environment variable: {line}\n{string}") continue # TODO: why? if kv[0] == "Path": kv[0] = "PATH" env[kv[0]] = kv[1] return env @staticmethod def _callVCVER(version : int, args : []=None, native : bool=True, prerelease : bool=False) -> str: if not args: args = [] vswhere = os.path.join(CraftCore.standardDirs.craftBin(), "3rdparty", "vswhere", "vswhere.exe") command = [vswhere, "-property", "installationPath", "-nologo", "-latest"] if prerelease: command += ["-prerelease"] if version: command += ["-version", f"[{version},{version+1})"] if version < 15: command.append("-legacy") else: if not args: args = ["-products", "*"] if native: # this fails with express versions args += ["-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64"] return SetupHelper._getOutput(command + args)[1] @staticmethod def getMSVCEnv(version : int=0, architecture="x86", toolset=None, native=True) -> str: if native: architectures = {"x86": "amd64_x86", "x64": "amd64"} else: architectures = {"x86": "x86", "x64": "x86_amd64"} args = architectures[architecture] path = "" # we prefer newer compiler that provide legacy toolchains if version and version < 16: # are we using msvc2017 with "VC++ 2015.3 v14.00 (v140) toolset for desktop" component = "Microsoft.VisualStudio.Component.VC." if version == 14: component += str(CraftCore.compiler.getMsvcPlatformToolset()) else: component += f"v{CraftCore.compiler.getMsvcPlatformToolset()}.x86.x64" # todo directly get the correct version for v in [16, 15]: path = SetupHelper._callVCVER(v, args=["-products", "*", "-requires", component], native=native) if path: if not toolset: toolset = str(CraftCore.compiler.getMsvcPlatformToolset() / 10) break if toolset: args += f" -vcvars_ver={toolset}" if not path: path = SetupHelper._callVCVER(version, native=native) if not path: path = SetupHelper._callVCVER(version, native=native, prerelease=True) if path: log("Found MSVS only in a prerelease version. I will use that.") if not path: log("Unable to locate Visual Studio. Please install it with the C++ component. Aborting.", critical=True) path = os.path.join(path, "VC") if not os.path.exists(os.path.join(path, "vcvarsall.bat")): path = os.path.join(path, "Auxiliary", "Build") path = os.path.join(path, "vcvarsall.bat") if not os.path.isfile(path): log(f"Failed to setup msvc compiler.\n" f"{path} does not exist.", critical=True) status, result = SetupHelper._getOutput(f"\"{path}\" {args} > NUL && set", shell=True) if status != 0: log(f"Failed to setup msvc compiler.\n" f"Command: {result} ", critical=True) return SetupHelper.stringToEnv(result) def getEnv(self): if CraftCore.compiler.isMSVC(): return SetupHelper.getMSVCEnv(CraftCore.compiler.getInternalVersion(), CraftCore.compiler.architecture, CraftCore.compiler.msvcToolset, CraftCore.compiler.isNative()) elif CraftCore.compiler.isIntel(): architectures = {"x86": "ia32", "x64": "intel64"} programFiles = os.getenv("ProgramFiles(x86)") or os.getenv("ProgramFiles") status, result = SetupHelper._getOutput( "\"%s\\Intel\\Composer XE\\bin\\compilervars.bat\" %s > NUL && set" % ( programFiles, architectures[CraftCore.compiler.architecture]), shell=True) if status != 0: log("Failed to setup intel compiler", critical=True) return SetupHelper.stringToEnv(result) return os.environ def setXDG(self): self.prependEnvVar("XDG_DATA_DIRS", [os.path.join(CraftCore.standardDirs.craftRoot(), "share")]) if OsUtils.isUnix(): self.prependEnvVar("XDG_CONFIG_DIRS", [os.path.join(CraftCore.standardDirs.craftRoot(), "etc", "xdg")]) self.addEnvVar("XDG_DATA_HOME", os.path.join(CraftCore.standardDirs.craftRoot(), "home", os.getenv("USER"), ".local5", "share")) self.addEnvVar("XDG_CONFIG_HOME", os.path.join(CraftCore.standardDirs.craftRoot(), "home", os.getenv("USER"), ".config")) self.addEnvVar("XDG_CACHE_HOME", os.path.join(CraftCore.standardDirs.craftRoot(), "home", os.getenv("USER"), ".cache")) def _setupUnix(self): if CraftCore.compiler.isLinux: self.prependEnvVar("LDFLAGS", "-Wl,-rpath,'$ORIGIN/../lib'", sep=" ") self.prependEnvVar("LD_LIBRARY_PATH", [os.path.join(CraftCore.standardDirs.craftRoot(), "lib"), os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "x86_64-linux-gnu")]) self.prependEnvVar("BISON_PKGDATADIR", os.path.join(CraftCore.standardDirs.craftRoot(), "share", "bison")) self.prependEnvVar("M4", os.path.join(CraftCore.standardDirs.craftRoot(), "dev-utils", "bin", "m4")) def _setupMac(self): #self.prependEnvVar("DYLD_LIBRARY_PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "lib")) # we will later replace the hard coded path in BuildSystemBase.internalPostInstall self.prependEnvVar("LDFLAGS", f"-Wl,-rpath,{os.path.join(CraftCore.standardDirs.craftRoot(), 'lib')}", sep=" ") self.prependEnvVar("BISON_PKGDATADIR", os.path.join(CraftCore.standardDirs.craftRoot(), "share", "bison")) self.prependEnvVar("M4", os.path.join(CraftCore.standardDirs.craftRoot(), "dev-utils", "bin", "m4")) + self.addEnvVar("MACOSX_DEPLOYMENT_TARGET", CraftCore.settings.get("General", "MacDeploymentTarget", "10.13")) dbusInstalled = CraftCore.installdb.isInstalled("libs/dbus") if dbusInstalled: serviceAgent = os.path.join(CraftCore.standardDirs.craftRoot(), "Library", "LaunchAgents", "org.freedesktop.dbus-session.plist") if os.path.exists(serviceAgent): SetupHelper._getOutput(["launchctl", "load", "-Fw", serviceAgent]) def _setupWin(self): if not "HOME" in os.environ: self.addEnvVar("HOME", os.getenv("USERPROFILE")) if CraftCore.compiler.isMinGW(): if not CraftCore.settings.getboolean("QtSDK", "Enabled", "false"): if CraftCore.compiler.isX86(): self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "mingw", "bin")) else: self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "mingw64", "bin")) else: compilerName = CraftCore.settings.get("QtSDK", "Compiler") compilerMap = {"mingw53_32": "mingw530_32"} self.prependEnvVar("PATH", os.path.join(CraftCore.settings.get("QtSDK", "Path"), "Tools", compilerMap.get(compilerName, compilerName), "bin")) if CraftCore.settings.getboolean("QtSDK", "Enabled", "false"): self.prependEnvVar("PATH", os.path.join(CraftCore.settings.get("QtSDK", "Path"), CraftCore.settings.get("QtSDK", "Version"), CraftCore.settings.get("QtSDK", "Compiler"), "bin")) if CraftCore.compiler.isMinGW(): if not CraftCore.settings.getboolean("QtSDK", "Enabled", "false"): if CraftCore.compiler.isX86(): self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "mingw", "bin")) else: self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "mingw64", "bin")) else: compilerName = CraftCore.settings.get("QtSDK", "Compiler") compilerMap = {"mingw53_32": "mingw530_32"} self.prependEnvVar("PATH", os.path.join(CraftCore.settings.get("QtSDK", "Path"), "Tools", compilerMap.get(compilerName, compilerName), "bin")) def setupEnvironment(self): originaleEnv = CaseInsensitiveDict(os.environ) for var, value in CraftCore.settings.getSection("Environment"): # set and override existing values # the ini is case insensitive so sections are lowercase.... self.addEnvVar(var.upper(), value) self.prependEnvVar("PATH", os.path.dirname(sys.executable)) os.environ.update(self.getEnv()) self.addEnvVar("KDEROOT", CraftCore.standardDirs.craftRoot()) self.addEnvVar("SSL_CERT_FILE", os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")) self.addEnvVar("REQUESTS_CA_BUNDLE", os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")) if CraftCore.settings.getboolean("Compile", "UseCCache", False): self.addEnvVar("CCACHE_DIR", CraftCore.settings.get("Paths", "CCACHE_DIR", os.path.join(CraftCore.standardDirs.craftRoot(), "build", "CCACHE"))) if CraftCore.settings.getboolean("QtSDK", "Enabled", "false"): sdkPath = os.path.join(CraftCore.settings.get("QtSDK", "Path"), CraftCore.settings.get("QtSDK", "Version"), CraftCore.settings.get("QtSDK", "Compiler"), "bin") if not os.path.exists(sdkPath): log(f"Please ensure that you have installed the Qt SDK in {sdkPath}", critical=True) self.prependEnvVar("PATH", sdkPath) if OsUtils.isWin(): self._setupWin() elif OsUtils.isMac(): self._setupMac() else: self._setupUnix() PKG_CONFIG_PATH = collections.OrderedDict.fromkeys([os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "pkgconfig")]) if "PKG_CONFIG_PATH" in originaleEnv: PKG_CONFIG_PATH.update(collections.OrderedDict.fromkeys(originaleEnv["PKG_CONFIG_PATH"].split(os.path.pathsep))) else: pkgCOnfig = shutil.which("pkg-config", path=originaleEnv["PATH"]) if pkgCOnfig: out = self._getOutput("pkg-config --variable pc_path pkg-config", shell=True) if out[0] == 0: PKG_CONFIG_PATH.update(collections.OrderedDict.fromkeys(out[1].split(os.path.pathsep))) self.prependEnvVar("PKG_CONFIG_PATH", os.path.pathsep.join(PKG_CONFIG_PATH.keys())) self.prependEnvVar("QT_PLUGIN_PATH", [os.path.join(CraftCore.standardDirs.craftRoot(), "plugins"), os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "plugins"), os.path.join(CraftCore.standardDirs.craftRoot(), "lib64", "plugins"), os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "x86_64-linux-gnu", "plugins"), os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "plugin") ]) self.prependEnvVar("QML2_IMPORT_PATH", [os.path.join(CraftCore.standardDirs.craftRoot(), "qml"), os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "qml"), os.path.join(CraftCore.standardDirs.craftRoot(), "lib64", "qml"), os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "x86_64-linux-gnu", "qml") ]) self.prependEnvVar("QML_IMPORT_PATH", os.environ["QML2_IMPORT_PATH"]) self.prependEnvVar("QT_DATA_DIRS", CraftCore.standardDirs.locations.data) if CraftCore.settings.getboolean("General", "UseSandboxConfig", True): self.setXDG() self.prependEnvVar("PATH", CraftCore.standardDirs.craftBin()) # add python site packages to pythonpath self.prependEnvVar("PYTHONPATH", os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "site-packages")) # prepend our venv python self.prependEnvVar("PATH", [os.path.join(CraftCore.standardDirs.etcDir(), f"virtualenv", "3", "Scripts" if CraftCore.compiler.isWindows else "bin"), os.path.join(CraftCore.standardDirs.etcDir(), f"virtualenv", "2", "Scripts" if CraftCore.compiler.isWindows else "bin")]) # make sure that craftroot bin is the first to look for dlls etc self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "bin")) self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "dev-utils", "bin")) if CraftCore.compiler.isClang(): if OsUtils.isUnix(): self.addEnvVar("CC", "/usr/bin/clang") self.addEnvVar("CXX", "/usr/bin/clang++") else: if CraftCore.compiler.isMSVC(): self.addEnvVar("CC", "clang-cl") self.addEnvVar("CXX", "clang-cl") else: self.addEnvVar("CC", "clang") self.addEnvVar("CXX", "clang") elif CraftCore.compiler.isGCC(): if not CraftCore.compiler.isNative() and CraftCore.compiler.isX86(): self.addEnvVar("CC", "gcc -m32") self.addEnvVar("CXX", "g++ -m32") self.addEnvVar("AS", "gcc -c -m32") else: self.addEnvVar("CC", "gcc") self.addEnvVar("CXX", "g++") if CraftCore.settings.getboolean("General", "AllowAnsiColor", False): # different non standard env switches self.addEnvVar("CLICOLOR_FORCE", "1") self.addEnvVar("CLICOLOR", "1") self.addEnvVar("ANSICON", "1") if CraftCore.compiler.isClang() and CraftCore.compiler.isMSVC(): self.prependEnvVar("CFLAGS", "-fcolor-diagnostics -fansi-escape-codes", sep=" ") self.prependEnvVar("CXXFLAGS", "-fcolor-diagnostics -fansi-escape-codes", sep=" ") elif CraftCore.compiler.isGCCLike(): self.prependEnvVar("CFLAGS", "-fdiagnostics-color=always", sep=" ") self.prependEnvVar("CXXFLAGS", "-fdiagnostics-color=always", sep=" ") if OsUtils.isWin(): self.addEnvVar("TERM", "xterm-color") # pretend to be a common smart terminal def printEnv(self): self.setupEnvironment() for key, val in os.environ.items(): if "\n" in val: log(f"Not adding ${key} to environment since it contains " "a newline character and that breaks craftenv.sh") continue if key.startswith("BASH_FUNC_"): continue # weird protected env vars if key in {"PROFILEREAD"}: continue CraftCore.log.info(f"{key}={val}") @property def version(self): return CraftCore.settings.version helper = SetupHelper() if __name__ == '__main__': helper.run() diff --git a/bin/shells.py b/bin/shells.py index dc74a48f3..5cb44ce4d 100644 --- a/bin/shells.py +++ b/bin/shells.py @@ -1,210 +1,206 @@ #!/usr/bin/env python """ provides shells """ import platform import subprocess import sys from CraftCore import CraftCore from Blueprints.CraftVersion import CraftVersion from CraftOS.osutils import OsUtils import utils import os import shutil class BashShell(object): def __init__(self): self._environment = {} self._useMSVCCompatEnv = False @property def useMSVCCompatEnv(self): return self._useMSVCCompatEnv @useMSVCCompatEnv.setter def useMSVCCompatEnv(self, b): self._useMSVCCompatEnv = b self._environment = {} @property def environment(self): if not self._environment: mergeroot = self.toNativePath(CraftCore.standardDirs.craftRoot()) ldflags = f" -L{mergeroot}/lib " cflags = f" -I{mergeroot}/include " if CraftCore.compiler.isMacOS: # Only look for includes/libraries in the XCode SDK on MacOS to avoid errors with # libraries installed by homebrew (causes errors e.g. with iconv since headers will be # found in /usr/local/include first but libraries are searched for in /usr/lib before # /usr/local/lib. See https://langui.sh/2015/07/24/osx-clang-include-lib-search-paths/ - if CraftCore.compiler.macUseSDK: - sdkPath = CraftCore.cache.getCommandOutput("xcrun", "--show-sdk-path")[1].strip() - deploymentFlag = "-mmacosx-version-min=" + CraftCore.compiler.macOSDeploymentTarget - cflags = f" -isysroot {sdkPath} {deploymentFlag}" + cflags - # See https://github.com/Homebrew/homebrew-core/issues/2674 for the -no_weak_imports flag - ldflags = f" -isysroot {sdkPath} {deploymentFlag} -Wl,-no_weak_imports" + ldflags - # Note: MACOSX_DEPLOYMENT_TARGET is set in utils.system() so doesn't need to be set here - else: - # Ensure that /usr/include comes before /usr/local/include in the header search path to avoid - # pulling in headers from /usr/local/include (e.g. installed by homebrew) that will cause - # linker errors later. - cflags = " -isystem /usr/include " + cflags + # Ensure that /usr/include comes before /usr/local/include in the header search path to avoid + # pulling in headers from /usr/local/include (e.g. installed by homebrew) that will cause + # linker errors later. + sdkPath = CraftCore.cache.getCommandOutput("xcrun", "--show-sdk-path")[1].strip() + deploymentFlag = f"-mmacosx-version-min={os.environ['MACOSX_DEPLOYMENT_TARGET']}" + cflags = f" -isysroot {sdkPath} {deploymentFlag} {cflags} -isystem /usr/include" + # See https://github.com/Homebrew/homebrew-core/issues/2674 for the -no_weak_imports flag + ldflags = f" -isysroot {sdkPath} {deploymentFlag} -Wl,-no_weak_imports {ldflags}" if CraftCore.compiler.isMSVC(): # based on Windows-MSVC.cmake if self.buildType == "Release": cflags += " -MD -O2 -Ob2 -DNDEBUG " elif self.buildType == "RelWithDebInfo": cflags += " -MD -Zi -O2 -Ob1 -DNDEBUG " ldflags += " -debug " elif self.buildType == "Debug": cflags += " -MDd -Zi -Ob0 -Od " ldflags += " -debug -pdbtype:sept " else: if self.buildType == "Release": cflags += " -O3 -DNDEBUG " if self.buildType == "RelWithDebInfo": cflags += " -O2 -g -DNDEBUG " elif self.buildType == "Debug": cflags += " -O0 -g3 " if OsUtils.isWin(): def convertPath(path : str): return ":".join([self.toNativePath(p) for p in path.split(os.path.pathsep)]) path = "/usr/local/bin:/usr/bin:/bin:/usr/bin/site_perl:/usr/bin/vendor_perl:/usr/bin/core_perl" if CraftCore.compiler.isMinGW(): gcc = shutil.which("gcc") if gcc: path = f"{self.toNativePath(os.path.dirname(gcc))}:{path}" elif CraftCore.compiler.isMSVC(): path = f"{self.toNativePath(os.path.dirname(shutil.which('cl')))}:{path}" self._environment["PATH"] = f"{path}:{convertPath(os.environ['PATH'])}" self._environment["PKG_CONFIG_PATH"] = convertPath(os.environ["PKG_CONFIG_PATH"]) if "make" in self._environment: del self._environment["make"] # MSYSTEM is used by uname if CraftCore.compiler.isMinGW(): self._environment["MSYSTEM"] = f"MINGW{CraftCore.compiler.bits}_CRAFT" elif CraftCore.compiler.isMSVC(): self._environment["MSYSTEM"] = f"MSYS{CraftCore.compiler.bits}_CRAFT" if self.useMSVCCompatEnv and CraftCore.compiler.isMSVC(): automake = [] for d in os.scandir(os.path.join(os.path.dirname(self._findBash()), "..", "share")): if d.name.startswith("automake"): automake += [(d.name.rsplit("-")[1], os.path.realpath(d.path))] automake.sort(key=lambda x: CraftVersion(x[0])) latestAutomake = automake[-1][1] if False: cl = "clang-cl" else: cl = "cl" clWrapper = self.toNativePath(os.path.join(latestAutomake, "compile")) self._environment["LD"] = "link -nologo" self._environment["CC"] = f"{clWrapper} {cl} -nologo" self._environment["CXX"] = self._environment["CC"] self._environment["CPP"] = f"{cl} -nologo -EP" self._environment["CXXCPP"] = self._environment["CPP"] self._environment["NM"] = "dumpbin -symbols" self._environment["RC"] = f"windres -O COFF --target={'pe-i386' if CraftCore.compiler.isX86() else 'pe-x86-64'} --preprocessor='cl -nologo -EP -DRC_INVOKED -DWINAPI_FAMILY=0'" self._environment["STRIP"] = ":" self._environment["RANLIB"] = ":" self._environment["F77"] = "no" self._environment["FC"] = "no" cflags += (" -GR -W3 -EHsc" # dynamic and exceptions enabled " -D_USE_MATH_DEFINES -DWIN32_LEAN_AND_MEAN -DNOMINMAX -D_CRT_SECURE_NO_WARNINGS" " -wd4005" # don't warn on redefine " -wd4996" # The POSIX name for this item is deprecated. ) if CraftCore.compiler.getMsvcPlatformToolset() > 120: cflags += " -FS" self._environment["CFLAGS"] = os.environ.get("CFLAGS", "").replace("$", "$$") + cflags self._environment["CXXFLAGS"] = os.environ.get("CXXFLAGS", "").replace("$", "$$") + cflags self._environment["LDFLAGS"] = os.environ.get("LDFLAGS", "").replace("$", "$$") + ldflags return self._environment @property def buildType(self): return CraftCore.settings.get("Compile", "BuildType", "RelWithDebInfo") @staticmethod def toNativePath(path): if OsUtils.isWin(): return OsUtils.toMSysPath(path) else: return path def _findBash(self): if OsUtils.isWin(): msysdir = CraftCore.standardDirs.msysDir() bash = CraftCore.cache.findApplication("bash", os.path.join(msysdir, "usr", "bin")) else: bash = CraftCore.cache.findApplication("bash") if not bash: CraftCore.log.critical("Failed to detect bash") return bash def execute(self, path, cmd, args="", **kwargs): # try to locate the command tmp = CraftCore.cache.findApplication(cmd) if tmp: cmd = tmp if CraftCore.compiler.isWindows: command = f"{self._findBash()} -c \"{self.toNativePath(cmd)} {args}\"" else: command = f"{self.toNativePath(cmd)} {args}" env = dict(os.environ) env.update(self.environment) env.update(kwargs.get("env", {})) return utils.system(command, cwd=path, env=env,**kwargs) def login(self): if CraftCore.compiler.isMSVC(): self.useMSVCCompatEnv = True return self.execute(os.curdir, self._findBash(), "-i", displayProgress=True) class Powershell(object): def __init__(self): self.pwsh = CraftCore.cache.findApplication("pwsh") if not self.pwsh: if platform.architecture()[0] == "32bit": self.pwsh = CraftCore.cache.findApplication("powershell", os.path.join(os.environ["WINDIR"], "sysnative", "WindowsPowerShell", "v1.0" )) if not self.pwsh: self.pwsh = CraftCore.cache.findApplication("powershell") if not self.pwsh: CraftCore.log.warning("Failed to detect powershell") def quote(self, s : str) -> str: return f"'{s}'" def execute(self, args :[str]) -> bool: return utils.system([self.pwsh, "-NoProfile", "-ExecutionPolicy", "ByPass", "-Command"] + args) def main(): shell = BashShell() shell.login() def testColor(): shell = BashShell() shell.execute(CraftCore.standardDirs.craftRoot(), os.path.join(CraftCore.standardDirs.craftBin(), "data", "ansi_color.sh")) if __name__ == '__main__': if len(sys.argv) > 1: if sys.argv[1] == "color": testColor() else: main() diff --git a/bin/utils.py b/bin/utils.py index 7a406103b..53b661a45 100644 --- a/bin/utils.py +++ b/bin/utils.py @@ -1,1130 +1,1128 @@ # -*- coding: utf-8 -*- # copyright: # Holger Schroeder # Patrick Spendrin # Ralf Habacker # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import configparser import contextlib import glob import inspect import io import os import re import shlex import stat import shutil import sys import subprocess import tempfile from pathlib import Path import Notifier.NotificationLoader from Blueprints.CraftVersion import CraftVersion from CraftCore import CraftCore from CraftDebug import deprecated from CraftOS.osutils import OsUtils from CraftSetupHelper import SetupHelper from CraftStandardDirs import CraftStandardDirs def abstract(): caller = inspect.getouterframes(inspect.currentframe())[1][3] raise NotImplementedError(caller + ' must be implemented in subclass') ### unpack functions def unpackFiles(downloaddir, filenames, workdir): """unpack (multiple) files specified by 'filenames' from 'downloaddir' into 'workdir'""" for filename in filenames: if (not unpackFile(downloaddir, filename, workdir)): return False return True def unpackFile(downloaddir, filename, workdir): """unpack file specified by 'filename' from 'downloaddir' into 'workdir'""" CraftCore.log.debug(f"unpacking this file: {filename}") if not filename: return True (shortname, ext) = os.path.splitext(filename) if ext == "": CraftCore.log.warning(f"unpackFile called on invalid file extension {filename}") return True if OsUtils.isWin() and not OsUtils.supportsSymlinks(): CraftCore.log.warning("Please enable Windows 10 development mode to enable support for symlinks.\n" "This will enable faster extractions.\n" "https://docs.microsoft.com/en-us/windows/uwp/get-started/enable-your-device-for-development") if CraftCore.cache.findApplication("7za"): # we use tar on linux not 7z, don't use tar on windows as it skips symlinks # test it with breeze-icons if (not OsUtils.isWin() or (OsUtils.supportsSymlinks() and CraftCore.cache.getVersion("7za", versionCommand="-version") >= "16") or not re.match("(.*\.tar.*$|.*\.tgz$)", filename)): return un7zip(os.path.join(downloaddir, filename), workdir, ext) try: shutil.unpack_archive(os.path.join(downloaddir, filename), workdir) except Exception as e: CraftCore.log.error(f"Failed to unpack {filename}", exc_info=e) return False return True def un7zip(fileName, destdir, flag=None): ciMode = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) createDir(destdir) kw = {} progressFlags = [] type = [] resolveSymlinks = False app = CraftCore.cache.findApplication("7za") if not ciMode and CraftCore.cache.checkCommandOutputFor(app, "-bs"): progressFlags = ["-bso2", "-bsp1"] kw["stderr"] = subprocess.PIPE if flag == ".7z": # Actually this is not needed for a normal archive. # But git is an exe file renamed to 7z and we need to specify the type. # Yes it is an ugly hack. type = ["-t7z"] if re.match("(.*\.tar.*$|.*\.tgz$)", fileName): if progressFlags: if ciMode: progressFlags = [] else: # print progress to stderr progressFlags = ["-bsp2"] kw["pipeProcess"] = subprocess.Popen([app, "x", fileName, "-so"] + progressFlags, stdout = subprocess.PIPE) if OsUtils.isWin(): resolveSymlinks = True if progressFlags: progressFlags = ["-bsp0"] command = [app, "x", "-si", f"-o{destdir}", "-ttar"] + progressFlags else: tar = CraftCore.cache.findApplication("tar") command = [tar, "--directory", destdir, "-xf", "-"] else: command = [app, "x", "-r", "-y", f"-o{destdir}", fileName] + type + progressFlags # While 7zip supports symlinks cmake 3.8.0 does not support symlinks return system(command, displayProgress=True, **kw) and (not resolveSymlinks or replaceSymlinksWithCopies(destdir)) def compress(archive : str, source : str) -> bool: ciMode = CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) def __7z(archive, source): archive = Path(archive) app = CraftCore.cache.findApplication("7za") kw = {} flags = [] if archive.suffix in {".appxsym", ".appxupload"}: flags.append("-tzip") if not ciMode and CraftCore.cache.checkCommandOutputFor(app, "-bs"): flags += ["-bso2", "-bsp1"] kw["stderr"] = subprocess.PIPE if CraftCore.compiler.isUnix: tar = CraftCore.cache.findApplication("tar") kw["pipeProcess"] = subprocess.Popen([tar, "-cf", "-", "-C", source, ".",], stdout=subprocess.PIPE) command = [app, "a", "-si", archive] + flags else: command = [app, "a", "-r", archive] + flags if isinstance(source, list): command += source elif os.path.isfile(source): command += [source] else: command += [os.path.join(source, "*")] return system(command, displayProgress=True, **kw) def __xz(archive, source): command = ["tar", "-cJf", archive, "-C"] if os.path.isfile(source): command += [source] else: command += [source, "."] return system(command) createDir(os.path.dirname(archive)) if os.path.isfile(archive): deleteFile(archive) if CraftCore.compiler.isUnix and archive.endswith(".tar.xz"): return __xz(archive, source) else: return __7z(archive, source) def system(cmd, displayProgress=False, logCommand=True, acceptableExitCodes=None, **kw): """execute cmd in a shell. All keywords are passed to Popen. stdout and stderr might be changed depending on the chosen logging options.""" return systemWithoutShell(cmd, displayProgress=displayProgress, logCommand=logCommand, acceptableExitCodes=acceptableExitCodes, **kw) def systemWithoutShell(cmd, displayProgress=False, logCommand=True, pipeProcess=None, acceptableExitCodes=None, **kw): """execute cmd. All keywords are passed to Popen. stdout and stderr might be changed depending on the chosen logging options. When the parameter "displayProgress" is True, stdout won't be logged to allow the display of progress bars.""" environment = kw.get("env", os.environ) cwd = kw.get("cwd", os.getcwd()) # if the first argument is not an absolute path replace it with the full path to the application if isinstance(cmd, list): # allow to pass other types, like ints or Path cmd = [str(x) for x in cmd] if pipeProcess: pipeProcess.args = [str(x) for x in pipeProcess.args] arg0 = cmd[0] if not "shell" in kw: kw["shell"] = False else: if not "shell" in kw: # use shell, arg0 might end up with "/usr/bin/svn" => needs shell so it can be executed kw["shell"] = True arg0 = shlex.split(cmd, posix=not CraftCore.compiler.isWindows)[0] matchQuoted = re.match("^\"(.*)\"$", arg0) if matchQuoted: CraftCore.log.warning(f"Please don't pass quoted paths to systemWithoutShell, app={arg0}") if not os.path.isfile(arg0) and not matchQuoted: app = CraftCore.cache.findApplication(arg0) else: app = arg0 if app: if isinstance(cmd, list): cmd[0] = app elif not matchQuoted: cmd = cmd.replace(arg0, f"\"{app}\"", 1) else: app = arg0 if logCommand: _logCommand = "" if pipeProcess: _logCommand = "{0} | ".format(" ".join(pipeProcess.args)) _logCommand += " ".join(cmd) if isinstance(cmd, list) else cmd CraftCore.debug.print("executing command: {0}".format(_logCommand)) if pipeProcess: CraftCore.log.debug(f"executing command: {pipeProcess.args!r} | {cmd!r}") else: CraftCore.log.debug(f"executing command: {cmd!r}") CraftCore.log.debug(f"CWD: {cwd!r}") CraftCore.log.debug(f"displayProgress={displayProgress}") - if CraftCore.compiler.isMacOS and CraftCore.compiler.macUseSDK: - environment["MACOSX_DEPLOYMENT_TARGET"] = CraftCore.compiler.macOSDeploymentTarget CraftCore.debug.logEnv(environment) if pipeProcess: kw["stdin"] = pipeProcess.stdout if not displayProgress or CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): stdout = kw.get('stdout', sys.stdout) kw['stderr'] = subprocess.STDOUT kw['stdout'] = subprocess.PIPE proc = subprocess.Popen(cmd, **kw) if pipeProcess: pipeProcess.stdout.close() for line in proc.stdout: if isinstance(stdout, io.TextIOWrapper): if CraftCore.debug.verbose() < 3: # don't print if we write the debug log to stdout anyhow stdout.buffer.write(line) stdout.flush() elif stdout == subprocess.DEVNULL: pass elif isinstance(stdout, io.StringIO): stdout.write(line.decode("UTF-8")) else: stdout.write(line) CraftCore.log.debug("{app}: {out}".format(app=app, out=line.rstrip())) else: proc = subprocess.Popen(cmd, **kw) if pipeProcess: pipeProcess.stdout.close() if proc.stderr: for line in proc.stderr: CraftCore.log.debug("{app}: {out}".format(app=app, out=line.rstrip())) proc.communicate() proc.wait() if acceptableExitCodes is None: ok = proc.returncode == 0 else: ok = proc.returncode in acceptableExitCodes if not ok: msg = f"Command {cmd} failed with exit code {proc.returncode}" if not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): CraftCore.log.debug(msg) else: CraftCore.log.info(msg) return ok def cleanDirectory(directory): CraftCore.log.debug("clean directory %s" % directory) if os.path.exists(directory): # don't delete containg directrory as it might be a symlink and replacing it with a folder # breaks the behaviour with os.scandir(directory) as scan: for f in scan: if f.is_dir(): if not OsUtils.rmDir(f.path, force=True): return False else: if not OsUtils.rm(f.path, force=True): return False else: return createDir(directory) def getVCSType(url): """ return the type of the vcs url """ if not url: return "" if isGitUrl(url): return "git" elif isSvnUrl(url): return "svn" elif url.startswith("[hg]"): return "hg" ## \todo complete more cvs access schemes elif url.find("pserver:") >= 0: return "cvs" else: return "" def isGitUrl(Url): """ this function returns true, if the Url given as parameter is a git url: it either starts with git:// or the first part before the first '|' ends with .git or if the url starts with the token [git] """ if Url.startswith('git://'): return True # split away branch and tags splitUrl = Url.split('|') if splitUrl[0].endswith(".git"): return True if Url.startswith("[git]"): return True return False def isSvnUrl(url): """ this function returns true, if the Url given as parameter is a svn url """ if url.startswith("[svn]"): return True elif url.find("://") == -1: return True elif url.find("svn:") >= 0 or url.find("https:") >= 0 or url.find("http:") >= 0: return True return False def splitVCSUrl(Url): """ this function splits up an url provided by Url into the server name, the path, a branch or tag; it will return a list with 3 strings according to the following scheme: git://servername/path.git|4.5branch|v4.5.1 will result in ['git://servername:path.git', '4.5branch', 'v4.5.1'] This also works for all other dvcs""" splitUrl = Url.split('|') if len(splitUrl) < 3: c = [x for x in splitUrl] for dummy in range(3 - len(splitUrl)): c.append('') else: c = splitUrl[0:3] return c def replaceVCSUrl(Url): """ this function should be used to replace the url of a server this comes in useful if you e.g. need to switch the server url for a push url on gitorious.org """ configfile = os.path.join(CraftStandardDirs.etcBlueprintDir(), "..", "crafthosts.conf") replacedict = dict() # FIXME handle svn/git usernames and settings with a distinct naming # todo WTF if (("General", "KDESVNUSERNAME") in CraftCore.settings and CraftCore.settings.get("General", "KDESVNUSERNAME") != "username"): replacedict["git://git.kde.org/"] = "git@git.kde.org:" if os.path.exists(configfile): config = configparser.ConfigParser() config.read(configfile) # add the default KDE stuff if the KDE username is set. for section in config.sections(): host = config.get(section, "host") replace = config.get(section, "replace") replacedict[host] = replace for host in list(replacedict.keys()): if not Url.find(host) == -1: Url = Url.replace(host, replacedict[host]) break return Url def createImportLibs(dll_name, basepath): """creating the import libraries for the other compiler(if ANSI-C libs)""" dst = os.path.join(basepath, "lib") if (not os.path.exists(dst)): os.mkdir(dst) # check whether the required binary tools exist HAVE_GENDEF = CraftCore.cache.findApplication("gendef") is not None USE_GENDEF = HAVE_GENDEF HAVE_LIB = CraftCore.cache.findApplication("lib") is not None HAVE_DLLTOOL = CraftCore.cache.findApplication("dlltool") is not None CraftCore.log.debug(f"gendef found: {HAVE_GENDEF}") CraftCore.log.debug(f"gendef used: {USE_GENDEF}") CraftCore.log.debug(f"lib found: {HAVE_LIB}") CraftCore.log.debug(f"dlltool found: {HAVE_DLLTOOL}") dllpath = os.path.join(basepath, "bin", "%s.dll" % dll_name) defpath = os.path.join(basepath, "lib", "%s.def" % dll_name) exppath = os.path.join(basepath, "lib", "%s.exp" % dll_name) imppath = os.path.join(basepath, "lib", "%s.lib" % dll_name) gccpath = os.path.join(basepath, "lib", "%s.dll.a" % dll_name) if not HAVE_GENDEF and os.path.exists(defpath): HAVE_GENDEF = True USE_GENDEF = False if not HAVE_GENDEF: CraftCore.log.warning("system does not have gendef.exe") return False if not HAVE_LIB and not os.path.isfile(imppath): CraftCore.log.warning("system does not have lib.exe (from msvc)") if not HAVE_DLLTOOL and not os.path.isfile(gccpath): CraftCore.log.warning("system does not have dlltool.exe") # create .def if USE_GENDEF: cmd = "gendef - %s -a > %s " % (dllpath, defpath) system(cmd) if (HAVE_LIB and not os.path.isfile(imppath)): # create .lib cmd = "lib /machine:x86 /def:%s /out:%s" % (defpath, imppath) system(cmd) if (HAVE_DLLTOOL and not os.path.isfile(gccpath)): # create .dll.a cmd = "dlltool -d %s -l %s -k" % (defpath, gccpath) system(cmd) if os.path.exists(defpath): os.remove(defpath) if os.path.exists(exppath): os.remove(exppath) return True def createSymlink(source, linkName, useAbsolutePath=False, targetIsDirectory=False): if not useAbsolutePath and os.path.isabs(linkName): srcPath = linkName srcPath = os.path.dirname(srcPath) source = os.path.relpath(source, srcPath) createDir(os.path.dirname(linkName)) CraftCore.log.debug(f"creating symlink: {linkName} -> {source}") try: os.symlink(source, linkName, targetIsDirectory) return True except Exception as e: CraftCore.log.warning(e) return False def createDir(path): """Recursive directory creation function. Makes all intermediate-level directories needed to contain the leaf directory""" if not os.path.lexists(path): CraftCore.log.debug(f"creating directory {path}") os.makedirs(path) return True def copyFile(src : Path, dest : Path, linkOnly=CraftCore.settings.getboolean("General", "UseHardlinks", False)): """ copy file from src to dest""" CraftCore.log.debug("copy file from %s to %s" % (src, dest)) src = Path(src) dest = Path(dest) if dest.is_dir(): dest = dest / src.name else: createDir(dest.parent) if os.path.lexists(dest): CraftCore.log.warning(f"Overriding:\t{dest} with\n\t\t{src}") if src == dest: CraftCore.log.error(f"Can't copy a file into itself {src}=={dest}") return False OsUtils.rm(dest, True) # don't link to links if linkOnly and not os.path.islink(src): try: os.link(src, dest) return True except: CraftCore.log.warning("Failed to create hardlink %s for %s" % (dest, src)) try: shutil.copy2(src, dest, follow_symlinks=False) except Exception as e: CraftCore.log.error(f"Failed to copy file:\n{src} to\n{dest}", exc_info=e) return False return True def copyDir(srcdir, destdir, linkOnly=CraftCore.settings.getboolean("General", "UseHardlinks", False), copiedFiles=None): """ copy directory from srcdir to destdir """ CraftCore.log.debug("copyDir called. srcdir: %s, destdir: %s" % (srcdir, destdir)) srcdir = Path(srcdir) if not srcdir.exists(): CraftCore.log.warning(f"copyDir called. srcdir: {srcdir} does not exists") return True destdir = Path(destdir) if not destdir.exists(): createDir(destdir) try: with os.scandir(srcdir) as scan: for entry in scan: dest = destdir / Path(entry.path).parent.relative_to(srcdir) / entry.name if entry.is_dir(): if entry.is_symlink(): # copy the symlinks without resolving them if not copyFile(entry.path, dest, linkOnly=False): return False if copiedFiles is not None: copiedFiles.append(str(dest)) else: if not copyDir(entry.path, dest, copiedFiles=copiedFiles): return False else: # symlinks to files are included in `files` if not copyFile(entry.path, dest,linkOnly=linkOnly): return False if copiedFiles is not None: copiedFiles.append(str(dest)) except Exception as e: CraftCore.log.error(f"Failed to copy dir:\n{srcdir} to\n{destdir}", exc_info=e) return False return True def globCopyDir(srcDir : str, destDir : str, pattern : [str], linkOnly=CraftCore.settings.getboolean("General", "UseHardlinks", False)) -> bool: files = [] for p in pattern: files.extend(glob.glob(os.path.join(srcDir, p), recursive=True)) for f in files: if not copyFile(f, os.path.join(destDir, os.path.relpath(f, srcDir)), linkOnly=linkOnly): return False return True def mergeTree(srcdir, destdir): """ moves directory from @p srcdir to @p destdir If a directory in @p destdir exists, just write into it """ srcdir = Path(srcdir) destdir = Path(destdir) if not createDir(destdir): return False CraftCore.log.debug(f"mergeTree called. srcdir: {srcdir}, destdir: {destdir}") if srcdir.samefile(destdir): CraftCore.log.critical(f"mergeTree called on the same directory srcdir: {srcdir}, destdir: {destdir}") return False with os.scandir(srcdir) as scan: for src in scan: dest = destdir / src.name if Path(src.path) in dest.parents: CraftCore.log.info(f"mergeTree: skipping moving of {src.path} to {dest}") continue if dest.exists(): if dest.is_dir(): if dest.is_symlink(): if dest.samefile(src): CraftCore.log.info(f"mergeTree: skipping moving of {src.path} to {dest} as a symlink with the same destination already exists") continue else: CraftCore.log.critical(f"mergeTree failed: {src.path} and {dest} are both symlinks but point to different folders") return False if src.is_symlink() and not dest.is_symlink(): CraftCore.log.critical(f"mergeTree failed: how to merge symlink {src.path} into {dest}") return False if not src.is_symlink() and dest.is_symlink(): CraftCore.log.critical(f"mergeTree failed: how to merge folder {src.path} into symlink {dest}") return False if not mergeTree(src.path, dest): return False else: CraftCore.log.critical(f"mergeTree failed: how to merge folder {src.path} into file {dest}\n" f"If this error occured during packaging, consider extending the blacklist.") return False else: if not moveFile(src.path, destdir): return False if not os.listdir(srcdir): # Cleanup (only removing empty folders) return rmtree(srcdir) else: # we move a directory in one of its sub directories assert srcdir in destdir.parents return True @deprecated("moveFile") def moveDir(srcdir, destdir): """ move directory from srcdir to destdir """ return moveFile(srcdir, destdir) def moveFile(src, dest): """move file from src to dest""" CraftCore.log.debug("move file from %s to %s" % (src, dest)) try: shutil.move(src, dest, copy_function=lambda src, dest, *kw : shutil.copy2(src, dest, *kw, follow_symlinks=False)) except Exception as e: CraftCore.log.warning(e) return False return True def rmtree(directory): """ recursively delete directory """ CraftCore.log.debug("rmtree called. directory: %s" % (directory)) try: shutil.rmtree(directory, True) # ignore errors except Exception as e: CraftCore.log.warning(e) return False return True def deleteFile(fileName): """delete file """ if not os.path.exists(fileName): return False CraftCore.log.debug("delete file %s " % (fileName)) try: os.remove(fileName) except Exception as e: CraftCore.log.warning(e) return False return True def putenv(name, value): """set environment variable""" if value is None: msg = f"unset environment variable -- unset {name}" if name in os.environ: del os.environ[name] else: msg = f"set environment variable -- set {name}={value}" os.environ[name] = value if CraftCore.settings.getboolean("CraftDebug", "PrintPutEnv", False): CraftCore.log.info(msg) else: CraftCore.log.debug(msg) return True def applyPatch(sourceDir, f, patchLevel='0'): """apply single patch""" if os.path.isdir(f): # apply a whole dir of patches for patch in os.listdir(f): if not applyPatch(sourceDir, os.path.join(f, patch), patchLevel): return False return True with tempfile.TemporaryDirectory() as tmp: # rewrite the patch, the gnu patch on Windows is only capable # to read \r\n patches tmpPatch = os.path.join(tmp, os.path.basename(f)) with open(f, "rt", encoding="utf-8") as p: patchContent = p.read() with open(tmpPatch, "wt", encoding="utf-8") as p: p.write(patchContent) cmd = ["patch", "--ignore-whitespace", "-d", sourceDir, "-p", str(patchLevel), "-i", tmpPatch] result = system(cmd) if not result: CraftCore.log.warning(f"applying {f} failed!") return result def embedManifest(executable, manifest): ''' Embed a manifest to an executable using either the free kdewin manifest if it exists in dev-utils/bin or the one provided by the Microsoft Platform SDK if it is installed' ''' if not os.path.isfile(executable) or not os.path.isfile(manifest): # We die here because this is a problem with the blueprint files CraftCore.log.critical("embedManifest %s or %s do not exist" % (executable, manifest)) CraftCore.log.debug("embedding ressource manifest %s into %s" % \ (manifest, executable)) return system(["mt", "-nologo", "-manifest", manifest, f"-outputresource:{executable};1"]) def notify(title, message, alertClass=None, log=True): if log: CraftCore.debug.step(f"{title}: {message}") backends = CraftCore.settings.get("General", "Notify", "") if CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) or backends == "": return backends = Notifier.NotificationLoader.load(backends.split(";")) for backend in backends.values(): backend.notify(title, message, alertClass) def levenshtein(s1, s2): if len(s1) < len(s2): return levenshtein(s2, s1) if not s1: return len(s2) previous_row = range(len(s2) + 1) for i, c1 in enumerate(s1): current_row = [i + 1] for j, c2 in enumerate(s2): insertions = previous_row[ j + 1] + 1 # j+1 instead of j since previous_row and current_row are one character longer deletions = current_row[j] + 1 # than s2 substitutions = previous_row[j] + (c1 != c2) current_row.append(min(insertions, deletions, substitutions)) previous_row = current_row return previous_row[-1] def createShim(shim, target, args=None, guiApp=False, useAbsolutePath=False) -> bool: if not useAbsolutePath and os.path.isabs(target): target = os.path.relpath(target, os.path.dirname(shim)) createDir(os.path.dirname(shim)) if os.path.exists(shim): deleteFile(shim) if not args: args = [] elif isinstance(args, str): CraftCore.log.error("Please pass args as [str]") return system(f"kshimgen --create {shim} {target} -- {args}") return system(["kshimgen", "--create", shim, target , "--"] + args) def replaceSymlinksWithCopies(path, _replaceDirs=False): def resolveLink(path): while os.path.islink(path): toReplace = os.readlink(path) if not os.path.isabs(toReplace): path = os.path.join(os.path.dirname(path), toReplace) else: path = toReplace return path # symlinks to dirs are resolved after we resolved the files dirsToResolve = [] ok = True for root, _, files in os.walk(path): for svg in files: if not ok: return False path = os.path.join(root, svg) if os.path.islink(path): toReplace = resolveLink(path) if not os.path.exists(toReplace): CraftCore.log.error(f"Resolving {path} failed: {toReplace} does not exists.") continue if toReplace != path: if os.path.isdir(toReplace): if not _replaceDirs: dirsToResolve.append(path) else: os.unlink(path) ok = copyDir(toReplace, path) else: os.unlink(path) ok = copyFile(toReplace, path) while dirsToResolve: d = dirsToResolve.pop() if not os.path.exists(resolveLink(d)): CraftCore.log.warning(f"Delay replacement of {d}") dirsToResolve.append(d) continue if not replaceSymlinksWithCopies(os.path.dirname(d), _replaceDirs=True): return False return True def printProgress(percent): width, _ = shutil.get_terminal_size((80, 20)) width -= 20 # margin times = int(width / 100 * percent) sys.stdout.write( "\r[{progress}{space}]{percent}%".format(progress="#" * times, space=" " * (width - times), percent=percent)) sys.stdout.flush() class ScopedEnv(object): def __init__(self, env): self.oldEnv = {} for key, value in env.items(): self.oldEnv[key] = os.environ.get(key, None) putenv(key, value) def reset(self): for key, value in self.oldEnv.items(): putenv(key, value) def __enter__(self): return self def __exit__(self, exc_type, exc_value, trback): self.reset() def normalisePath(path): path = os.path.abspath(path) if OsUtils.isWin(): return path.replace("\\", "/") return path def configureFile(inFile : str, outFile : str, variables : dict) -> bool: CraftCore.log.debug(f"configureFile {inFile} -> {outFile}\n{variables}") configPatter = re.compile(r"@{([^{}]+)}") with open(inFile, "rt", encoding="UTF-8") as f: script = f.read() matches = configPatter.findall(script) if not matches: CraftCore.log.debug("Nothing to configure") return False while matches: for match in matches: val = variables.get(match, None) if val is None: linenUmber = 0 for line in script.split("\n"): if match in line: break linenUmber += 1 raise Exception(f"Failed to configure {inFile}: @{{{match}}} is not in variables\n" f"{linenUmber}:{line}") script = script.replace(f"@{{{match}}}", str(val)) matches = configPatter.findall(script) os.makedirs(os.path.dirname(outFile), exist_ok=True) with open(outFile, "wt", encoding="UTF-8") as f: f.write(script) return True def limitCommandLineLength(command : [str], args : [str]) -> [[str]]: # the actual limit is hard to get in python so lets just use a working random size SIZE = 1024 * 4 out = [] commandSize = sum(map(len, command)) if commandSize >= SIZE: CraftCore.log.error("Failed to compute command, command too long") return [] currentSize = commandSize tmp = [] for a in args: le = len(a) if currentSize + le >= SIZE: out.append(command + tmp) tmp = [] currentSize = commandSize tmp.append(a) currentSize += le if tmp: out.append(command + tmp) return out def sign(fileNames : [str]) -> bool: if not CraftCore.settings.getboolean("CodeSigning", "Enabled", False): return True if not CraftCore.compiler.isWindows: CraftCore.log.warning("Code signing is currently only supported on Windows") return True signTool = CraftCore.cache.findApplication("signtool", forceCache=True) if not signTool: env = SetupHelper.getMSVCEnv() signTool = CraftCore.cache.findApplication("signtool", env["PATH"], forceCache=True) if not signTool: CraftCore.log.warning("Code signing requires a VisualStudio installation") return False subjectName = CraftCore.settings.get("CodeSigning", "CommonName") command = [signTool, "sign", "/n", subjectName, "/tr", "http://timestamp.digicert.com", "/td", "SHA256", "/fd", "SHA256", "/a"] certFile = CraftCore.settings.get("CodeSigning", "Certificate", "") if certFile: command += ["/f", certFile] if True or CraftCore.debug.verbose() > 0: command += ["/v"] else: command += ["/q"] for args in limitCommandLineLength(command, fileNames): if not system(args): return False return True def signMacApp(appPath : str): if not CraftCore.settings.getboolean("CodeSigning", "Enabled", False): return True unlockMacKeychain() developerIdApplication = CraftCore.settings.get("CodeSigning", "MacDeveloperIdApplication") # Recursively sign app if not system(["codesign", "-s", developerIdApplication, "--force", "--preserve-metadata=entitlements", "--verbose=4", "--deep", appPath]): return False ## Verify signature if not system(["codesign", "-dv", appPath]): return False if not system(["codesign", "--verify", "-v", appPath]): return False if not system(["spctl", "-a", "-t", "exec", "-vv", appPath]): return False ## Validate that the key used for signing the binary matches the expected TeamIdentifier ## needed to pass the SocketApi through the sandbox #if not utils.system("codesign -dv %s 2>&1 | grep 'TeamIdentifier=%s'" % (self.appPath, teamIdentifierFromConfig)): #return False return True def signMacPackage(packagePath : str): if not CraftCore.settings.getboolean("CodeSigning", "Enabled", False): return True unlockMacKeychain() if packagePath.endswith(".dmg"): # sign dmg developerIdApplication = CraftCore.settings.get("CodeSigning", "MacDeveloperIdApplication") if not system(["codesign", "--force", "--sign", developerIdApplication, packagePath]): return False # verify dmg signature if not system(["spctl", "-a", "-t", "open", "--context", "context:primary-signature", packagePath]): return False else: # sign pkg developerIdInstaller = CraftCore.settings.get("CodeSigning", "MacDeveloperIdInstaller") packagePathTmp = f"{packagePath}.sign" if not system(["productsign", "--sign", developerIdInstaller, packagePath, packagePathTmp]): return False os.rename(packagePathTmp, packagePath) return True def unlockMacKeychain(): password = CraftCore.settings.get("CodeSigning", "MacKeychainPassword") if not system(f"security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k \"{password}\" ~/Library/Keychains/login.keychain", stdout=subprocess.DEVNULL, logCommand=False): CraftCore.log.error("Cannot unlock keychain.") return False def isExecuatable(fileName : Path): fileName = Path(fileName) if CraftCore.compiler.isWindows: return fileName.suffix in os.environ["PATHEXT"] return os.access(fileName, os.X_OK) def isBinary(fileName : str) -> bool: # https://en.wikipedia.org/wiki/List_of_file_signatures fileName = Path(fileName) MACH_O_64 = b"\xCF\xFA\xED\xFE" ELF = b"\x7F\x45\x4C\x46" if fileName.is_symlink() or fileName.is_dir(): return False if CraftCore.compiler.isWindows: if fileName.suffix in {".dll", ".exe"}: return True else: if CraftCore.compiler.isMacOS and ".dSYM/" in str(fileName): return False if fileName.suffix in {".so", ".dylib"}: return True elif isExecuatable(fileName): if CraftCore.compiler.isMacOS: signature = MACH_O_64 elif CraftCore.compiler.isLinux: signature = ELF else: raise Exception("Unsupported platform") with open(fileName, "rb") as f: return f.read(len(signature)) == signature return False def getLibraryDeps(path): deps = [] if CraftCore.compiler.isMacOS: # based on https://github.com/qt/qttools/blob/5.11/src/macdeployqt/shared/shared.cpp infoRe = re.compile("^\\t(.+) \\(compatibility version (\\d+\\.\\d+\\.\\d+), "+ "current version (\\d+\\.\\d+\\.\\d+)\\)$") with io.StringIO() as log: if not system(["otool", "-L", path], stdout=log, logCommand=False): return [] lines = log.getvalue().strip().split("\n") lines.pop(0)# name of the library for line in lines: match = infoRe.match(line) if match: deps.append(match[1]) return deps def regexFileFilter(filename : os.DirEntry, root : str, pattern : [re]=None) -> bool: """ return False if file does not match pattern""" # use linux style seperators relFilePath = Path(filename.path).relative_to(root).as_posix() for pattern in pattern: if pattern.search(relFilePath): CraftCore.log.debug(f"regExDirFilter: {relFilePath} matches: {pattern.pattern}") return True return False def filterDirectoryContent(root, whitelist=lambda f, root: True, blacklist=lambda g, root: False, allowBadSymlinks=False, handleAppBundleAsFile=False): """ Traverse through a directory tree and return every filename that the function whitelist returns as true and which do not match blacklist entries """ if not os.path.exists(root): return dirs = [root] while dirs: path = dirs.pop() with os.scandir(path) as scan: for filePath in scan: if not allowBadSymlinks and filePath.is_symlink(): if Path(root) not in Path(filePath.path).resolve().parents: CraftCore.log.debug(f"filterDirectoryContent: skipping {filePath.path}, it is not located under {root}") continue if filePath.is_dir(follow_symlinks=False): # handle .app folders and dsym as files if not CraftCore.compiler.isMacOS or not (filePath.name.endswith(".dSYM") or handleAppBundleAsFile and filePath.name.endswith(".app")): dirs.append(filePath.path) continue if blacklist(filePath, root=root) and not whitelist(filePath, root=root): continue elif filePath.is_dir(): yield filePath.path elif filePath.is_file(): yield filePath.path elif filePath.is_symlink(): if not allowBadSymlinks: CraftCore.log.warning(f"{filePath.path} is an invalid link ({os.readlink(filePath.path)})") continue else: yield filePath.path else: CraftCore.log.warning(f"Unhandled case: {filePath}") raise Exception(f"Unhandled case: {filePath}") @contextlib.contextmanager def makeWritable(targetPath: Path): if isinstance(targetPath, str): targetPath = Path(targetPath) originalMode = targetPath.stat().st_mode wasWritable = bool(originalMode & stat.S_IWUSR) try: # ensure it is writable if not wasWritable: targetPath.chmod(originalMode | stat.S_IWUSR) yield targetPath finally: if not wasWritable: targetPath.chmod(originalMode) def getPDBForBinary(path :str) -> str: with open(path, "rb") as f: data = f.read() pdb = data.rfind(b".pdb") if pdb: return data[data.rfind(0x00, 0, pdb) + 1:pdb + 4].decode("utf-8") return "" def installShortcut(name : str, path : str, workingDir : str, icon : str, desciption : str): if not CraftCore.compiler.isWindows: return True from shells import Powershell pwsh = Powershell() shortcutPath = Path(os.environ["APPDATA"]) / f"Microsoft/Windows/Start Menu/Programs/Craft/{name}.lnk" shortcutPath.parent.mkdir(parents=True, exist_ok=True) return pwsh.execute([os.path.join(CraftCore.standardDirs.craftBin(), "install-lnk.ps1"), "-Path", pwsh.quote(path), "-WorkingDirectory", pwsh.quote(OsUtils.toNativePath(workingDir)), "-Name", pwsh.quote(shortcutPath), "-Icon", pwsh.quote(icon), "-Description", pwsh.quote(desciption)]) def strip(fileName): """strip debugging informations from shared libraries and executables - mingw only!!! """ if CraftCore.compiler.isMSVC() or not CraftCore.compiler.isGCCLike(): CraftCore.log.warning(f"Skipping stripping of {fileName} -- either disabled or unsupported with this compiler") return True fileName = Path(fileName) isBundle = False if CraftCore.compiler.isMacOS: bundleDir = list(filter(lambda x: x.name.endswith(".framework") or x.name.endswith(".app"), fileName.parents)) if bundleDir: suffix = "" # if we are a .app in a .framework we put the smbols in the same location if len(bundleDir) > 1: suffix = f"-{'.'.join([x.name for x in reversed(bundleDir[0:-1])])}" isBundle = True symFile = Path(f"{bundleDir[-1]}{suffix}.dSYM") else: symFile = Path(f"{fileName}.dSYM") else: symFile = Path(f"{fileName}.sym") if not isBundle and symFile.exists(): return True elif (symFile / "Contents/Resources/DWARF" / fileName.name).exists(): return True if CraftCore.compiler.isMacOS: return (system(["dsymutil", fileName, "-o", symFile]) and system(["strip", "-x", "-S", fileName])) else: return (system(["objcopy", "--only-keep-debug", fileName, symFile]) and system(["strip", "--strip-debug", "--strip-unneeded", fileName]) and system(["objcopy", "--add-gnu-debuglink", symFile, fileName]))