diff --git a/bin/CraftCore.py b/bin/CraftCore.py index 145e5887c..436176ddb 100644 --- a/bin/CraftCore.py +++ b/bin/CraftCore.py @@ -1,79 +1,76 @@ import importlib import logging +import os import sys # Add imports that cause a cyclic dependency in a not taken branch to make code completion work if False: from .CraftCompiler import CraftCompiler from .CraftDebug import CraftDebug from .CraftStandardDirs import CraftStandardDirs from .CraftConfig import CraftConfig from .Utils.CraftCache import CraftCache from .InstallDB import InstallDB # TODO: a more optimal solution would be to initialize all singletons in a # __init__.py but that would require massive refactoring as everything in bin/ -# is not part of a module wich could use such a __init__.py +# is not part of a module which could use such a __init__.py class AutoImport(object): - def __init__(self, name : str, module : str, className : str=None, function=None) -> None: + def __init__(self, name : str, module : str, className : str=None, function=None, member : str=None) -> None: self.name = name self.module = module self.className = className or module self.function = function + self.member = member def __getattribute__(self, name : str): _name = super().__getattribute__("name") _module = super().__getattribute__("module") _className = super().__getattribute__("className") _function = super().__getattribute__("function") + _member= super().__getattribute__("member") - mod = importlib.import_module(_module) - cls = getattr(mod, _className) - if _function: - func = getattr(cls, _function) - instance = func() + if _member: + # initialize with a member of another object + source = getattr(CraftCore, _name) + out = getattr(source, _member) + setattr(CraftCore, _member, out) + return getattr(out, name) else: - instance = cls() - setattr(CraftCore, _name, instance) - return instance.__getattribute__(name) + mod = importlib.import_module(_module) + cls = getattr(mod, _className) + if _function: + func = getattr(cls, _function) + instance = func() + else: + instance = cls() + setattr(CraftCore, _name, instance) + return instance.__getattribute__(name) def __str__(self): # force replacement self.__getattribute__ # TODO: find out why how self was replaced .... return self.__str__() class State(object): def __init__(self): # targets directly passed to craft self.directTargets = [] class CraftCore(object): debug = AutoImport("debug", "CraftDebug") # type: CraftDebug - log = None # type: logging.Logger + # log will be replaced once debug is loaded + log = AutoImport("debug", "CraftDebug", member="log") # type: logging.Logger standardDirs = AutoImport("standardDirs", "CraftStandardDirs") # type: CraftStandardDirs settings = AutoImport("settings", "CraftConfig") # type: CraftConfig cache = AutoImport("cache", "Utils.CraftCache", "CraftCache", "_loadInstance") # type: CraftCache compiler = AutoImport("compiler", "CraftCompiler") # type: CraftCompiler installdb = AutoImport("installdb", "InstallDB") # type: InstallDB # information about the current internal state of Craft state = State() - @classmethod - def registerObjectAlias(cls, name : str, source : str, obj : str) -> None: - """ - Allow to make a property of a singleton available through CraftCore - """ - if not hasattr(cls, source): - print(f"Unknown soruce {source}, please call CraftCore.registerInstance first", file=sys.stderr) - exit(1) - if not hasattr(cls, name): - print(f"Unknown property name {name}, please define CraftCore.{name}", file=sys.stderr) - exit(1) - if not hasattr(cls, name): - print(f"Unknown property {source.__class__}.{name}", file=sys.stderr) - exit(1) - if not getattr(cls, name): - setattr(cls, name, getattr(getattr(cls, source), obj)) +# make sure our environment is setup +import CraftSetupHelper \ No newline at end of file diff --git a/bin/CraftDebug.py b/bin/CraftDebug.py index f41b3c75b..6a8ec4e74 100644 --- a/bin/CraftDebug.py +++ b/bin/CraftDebug.py @@ -1,187 +1,185 @@ import functools import inspect import logging import logging.handlers import os import re import shutil import sys from CraftCore import CraftCore import CraftConfig class CraftDebug(object): def __init__(self): self.seenDeprecatedFunctions = set() self._handler = logging.StreamHandler(sys.stdout) self._log = logging.getLogger("craft") self._log.setLevel(logging.DEBUG) self._log.addHandler(self._handler) self._handler.setLevel(logging.INFO) logDir = CraftCore.settings.get("CraftDebug", "LogDir", os.path.expanduser("~/.craft/")) if not os.path.exists(logDir): os.makedirs(logDir) cleanNameRe = re.compile(r":?\\+|/+|:|;") logfileName = os.path.join(logDir, "log-%s.txt" % cleanNameRe.sub("_", CraftCore.settings._craftRoot())) try: fileHandler = logging.handlers.RotatingFileHandler(logfileName, mode="at+", maxBytes=10000000, backupCount=20) fileHandler.doRollover() fileHandler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) self._log.addHandler(fileHandler) fileHandler.setLevel(logging.DEBUG) except Exception as e: print(f"Failed to setup log file: {e}", file=sys.stderr) print(f"Right now we don't support running multiple Craft instances with the same configuration.", file=sys.stderr) self.log.debug("#" * self.lineWidth) self.log.debug("New log started: %s" % " ".join(sys.argv)) self.log.debug("Log is saved to: %s" % fileHandler.baseFilename) self.logEnv() self.setVerbose(0) @property def lineWidth(self): width, _ = shutil.get_terminal_size((80, 20)) return width def verbose(self): """return the value of the verbose level""" return self._verbosity def setVerbose(self, _verbose): self._verbosity = _verbose lvl = logging.INFO if 0 <= _verbose < 2: lvl = logging.INFO elif _verbose >= 2: lvl = logging.DEBUG elif _verbose == 0: lvl = logging.WARNING elif _verbose <= -1: lvl = logging.CRITICAL self._handler.setLevel(lvl) def step(self, message): self.log.info("*** %s ***" % message) def new_line(self): self.log.info("\n") def debug_line(self): self.log.info("=" * self.lineWidth) @property def log(self): return self._log def print(self, msg, file=sys.stdout, stack_info=False): if 0 <= self.verbose() < 2: print(msg, file=file if not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) else sys.stdout) self.log.debug(msg, stack_info=stack_info) else: self.log.debug(msg, stack_info=stack_info) def printOut(self, msg, file=sys.stdout): """ Should only be used to report independent of the verbosity level for example to print the installed files etc """ if self.verbose() < 2: print(msg, file=file if not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False) else sys.stdout) self.log.debug(msg) else: self.log.debug(msg) def logEnv(self, env=None): if CraftCore.settings.getboolean("CraftDebug", "LogEnvironment", True): if not env: env = os.environ self.log.debug( "Environment: \n" + "\n".join(f" {key}={value}" for key, value in env.items())) def trace(self, message): self.log.debug("craft trace: %s" % message) -CraftCore.registerObjectAlias("log", "debug", "log") - class TemporaryVerbosity(object): """Context handler for temporarily different verbosity""" def __init__(self, tempLevel): self.prevLevel = CraftCore.debug.verbose() CraftCore.debug.setVerbose(tempLevel) def __enter__(self): return self def __exit__(self, exc_type, exc_value, trback): CraftCore.debug.setVerbose(self.prevLevel) def deprecated(replacement=None): """ http://code.activestate.com/recipes/577819-deprecated-decorator/ Deprecated decorator. Author: Giampaolo Rodola' License: MIT A decorator which can be used to mark functions as deprecated. replacement is a callable that will be called with the same args as the decorated function. >>> @deprecated() ... def foo(x): ... return x ... >>> ret = foo(1) DeprecationWarning: foo is deprecated >>> ret 1 >>> >>> >>> def newfun(x): ... return 0 ... >>> @deprecated(newfun) ... def foo(x): ... return x ... >>> ret = foo(1) DeprecationWarning: foo is deprecated; use newfun instead >>> ret 0 >>> """ def outer(fun): msg = f"{fun.__name__} is deprecated" if replacement is not None: msg += f", use {replacement} instead" if fun.__doc__ is None: fun.__doc__ = msg @functools.wraps(fun) def inner(*args, **kwargs): _info = inspect.stack()[1] if not (_info.filename, _info.lineno) in CraftCore.debug.seenDeprecatedFunctions: CraftCore.debug.seenDeprecatedFunctions.add((_info.filename, _info.lineno)) if CraftCore.settings.getboolean("CraftDebug", "LogDeprecated", False): CraftCore.debug.print(msg, stack_info=True) else: CraftCore.log.debug(msg, stack_info=True) return fun(*args, **kwargs) return inner return outer if __name__ == "__main__": CraftCore.log.debug("debug: foo") CraftCore.log.info("info: foo") diff --git a/bin/CraftSetupHelper.py b/bin/CraftSetupHelper.py index b4c606c5f..6dd897195 100644 --- a/bin/CraftSetupHelper.py +++ b/bin/CraftSetupHelper.py @@ -1,409 +1,414 @@ # -*- coding: utf-8 -*- # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import argparse import collections -import subprocess -import copy +import os import platform +import shutil +import subprocess +import sys -from CraftConfig import * from CraftCore import CraftCore -from CraftDebug import CraftDebug from CraftOS.osutils import OsUtils -from CraftStandardDirs import CraftStandardDirs from Utils.CaseInsensitiveDict import CaseInsensitiveDict # The minimum python version for craft please edit here # if you add code that changes this requirement MIN_PY_VERSION = (3, 6, 0) def log(msg, critical=False): if critical or not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): CraftCore.debug.print(msg, sys.stderr) else: CraftCore.log.debug(msg) if critical: exit(1) if sys.version_info[0:3] < MIN_PY_VERSION: log("Error: Python too old!\n" "Craft needs at least Python Version %s.%s.%s\n" "Please install it and adapt your CraftSettings.ini" % MIN_PY_VERSION, critical=True) if not platform.machine().endswith("64"): log(f"Craft requires a 64bit operating system. Your are using: {platform.machine()}", critical=True) class SetupHelper(object): CraftVersion = "master" + NeedsSetup = "KDEROOT" not in os.environ def __init__(self, args=None): self.args = args if CraftCore.settings.getboolean("General", "AllowAnsiColor", False): OsUtils.enableAnsiColors() + if SetupHelper.NeedsSetup: + SetupHelper.NeedsSetup = False + self.checkForEvilApplication() + self.setupEnvironment() @staticmethod def _getOutput(command, shell=False): CraftCore.log.debug(f"SetupHelper._getOutput: {command}") p = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, universal_newlines=True, errors="backslashreplace") out = p.stdout.strip() CraftCore.log.debug(f"SetupHelper._getOutput: return {p.returncode} {out}") return p.returncode, out def run(self): parser = argparse.ArgumentParser() parser.add_argument("--get", action="store_true") parser.add_argument("--print-banner", action="store_true") parser.add_argument("--getenv", action="store_true") parser.add_argument("--setup", action="store_true") parser.add_argument("rest", nargs=argparse.REMAINDER) args = parser.parse_args() if args.get: default = "" if len(args.rest) == 3: default = args.rest[2] CraftCore.log.info(CraftCore.settings.get(args.rest[0], args.rest[1], default)) elif args.print_banner: self.printBanner() elif args.getenv: self.printEnv() elif args.setup: self.printEnv() self.printBanner() def checkForEvilApplication(self): blackList = [] if OsUtils.isWin(): blackList += ["sh", "gcc", "g++", "cpp"] for app in blackList: location = shutil.which(app) if location: location = os.path.dirname(location) if not CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False): log( f"Found \"{app}\" in your PATH: \"{location}\"\n" f"This application is known to cause problems with your configuration of Craft.\n" f"Please remove it from PATH or manually set a value for PATH in your CraftSettings.ini:\n" f"\n" f"[Environment]\n" f"PATH=" f"\n") else: path = collections.OrderedDict.fromkeys(os.environ["Path"].split(os.path.pathsep)) del path[location] self.addEnvVar("Path", os.path.pathsep.join(path)) - def printBanner(self): + @staticmethod + def printBanner(): def printRow(name, value): log(f"{name:20}: {value}") - printRow("Craft", CraftStandardDirs.craftRoot()) + printRow("Craft", CraftCore.standardDirs.craftRoot()) printRow("Version", SetupHelper.CraftVersion) printRow("ABI", CraftCore.compiler) - printRow("Download directory", CraftStandardDirs.downloadDir()) + printRow("Download directory", CraftCore.standardDirs.downloadDir()) def addEnvVar(self, key, val): os.environ[key] = val def prependEnvVar(self, key : str, var : str, sep : str=os.path.pathsep) -> None: if not type(var) == list: var = [var] if key in os.environ: env = var + os.environ[key].split(sep) var = list(collections.OrderedDict.fromkeys(env)) val = sep.join(var) CraftCore.log.debug(f"Setting {key}={val}") os.environ[key] = val @staticmethod def stringToEnv(string : str): for line in string.strip().split("\n"): kv = line.strip().split("=", 1) if len(kv) != 2: log(f"Failed to parse environment variable: {line}\n{string}") continue # TODO: why? if kv[0] == "Path": kv[0] = "PATH" os.environ[kv[0]] = kv[1] return os.environ @staticmethod def _callVCVER(version : int, args : []=None, native : bool=True) -> str: if not args: args = [] vswhere = os.path.join(CraftCore.standardDirs.craftBin(), "3rdparty", "vswhere", "vswhere.exe") command = [vswhere, "-property", "installationPath", "-nologo", "-latest"] if version: command += ["-version", f"[{version},{version+1})"] if version < 15: command.append("-legacy") else: if not args: args = ["-products", "*"] if native: # this fails with express versions args += ["-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64"] return SetupHelper._getOutput(command + args)[1] @staticmethod def getMSVCEnv(version=None, architecture="x86", toolset=None, native=True) -> str: if native: architectures = {"x86": "amd64_x86", "x64": "amd64"} else: architectures = {"x86": "x86", "x64": "x86_amd64"} args = architectures[architecture] path = "" if version == 14: # are we using msvc2017 with "VC++ 2015.3 v14.00 (v140) toolset for desktop" path = SetupHelper._callVCVER(15, args=["-products", "*", "-requires", "Microsoft.VisualStudio.Component.VC.140"], native=native) if path and not toolset: toolset = "14.0" if toolset: args += f" -vcvars_ver={toolset}" if not path: path = SetupHelper._callVCVER(version, native=native) if not path: log("Please ensure that you have installed the C++ component", critical=True) path = os.path.join(path, "VC") if not os.path.exists(os.path.join(path, "vcvarsall.bat")): path = os.path.join(path, "Auxiliary", "Build") path = os.path.join(path, "vcvarsall.bat") if not os.path.isfile(path): log(f"Failed to setup msvc compiler.\n" f"{path} does not exist.", critical=True) status, result = SetupHelper._getOutput(f"\"{path}\" {args} > NUL && set", shell=True) if status != 0: log(f"Failed to setup msvc compiler.\n" f"Command: {result} ", critical=True) return SetupHelper.stringToEnv(result) def getEnv(self): if CraftCore.compiler.isMSVC(): return SetupHelper.getMSVCEnv(CraftCore.compiler.getInternalVersion(), CraftCore.compiler.architecture, CraftCore.compiler.msvcToolset, CraftCore.compiler.isNative()) elif CraftCore.compiler.isIntel(): architectures = {"x86": "ia32", "x64": "intel64"} programFiles = os.getenv("ProgramFiles(x86)") or os.getenv("ProgramFiles") status, result = SetupHelper._getOutput( "\"%s\\Intel\\Composer XE\\bin\\compilervars.bat\" %s > NUL && set" % ( programFiles, architectures[CraftCore.compiler.architecture]), shell=True) if status != 0: log("Failed to setup intel compiler", critical=True) return SetupHelper.stringToEnv(result) return os.environ def setXDG(self): - self.prependEnvVar("XDG_DATA_DIRS", [os.path.join(CraftStandardDirs.craftRoot(), "share")]) + self.prependEnvVar("XDG_DATA_DIRS", [os.path.join(CraftCore.standardDirs.craftRoot(), "share")]) if OsUtils.isUnix(): - self.prependEnvVar("XDG_CONFIG_DIRS", [os.path.join(CraftStandardDirs.craftRoot(), "etc", "xdg")]) + self.prependEnvVar("XDG_CONFIG_DIRS", [os.path.join(CraftCore.standardDirs.craftRoot(), "etc", "xdg")]) self.addEnvVar("XDG_DATA_HOME", - os.path.join(CraftStandardDirs.craftRoot(), "home", os.getenv("USER"), ".local5", "share")) + os.path.join(CraftCore.standardDirs.craftRoot(), "home", os.getenv("USER"), ".local5", "share")) self.addEnvVar("XDG_CONFIG_HOME", - os.path.join(CraftStandardDirs.craftRoot(), "home", os.getenv("USER"), ".config")) + os.path.join(CraftCore.standardDirs.craftRoot(), "home", os.getenv("USER"), ".config")) self.addEnvVar("XDG_CACHE_HOME", - os.path.join(CraftStandardDirs.craftRoot(), "home", os.getenv("USER"), ".cache")) + os.path.join(CraftCore.standardDirs.craftRoot(), "home", os.getenv("USER"), ".cache")) def _setupUnix(self): if CraftCore.compiler.isLinux: self.prependEnvVar("LDFLAGS", "-Wl,-rpath,'$ORIGIN/../lib'", sep=" ") - self.prependEnvVar("LD_LIBRARY_PATH", [os.path.join(CraftStandardDirs.craftRoot(), "lib"), - os.path.join(CraftStandardDirs.craftRoot(), "lib", "x86_64-linux-gnu")]) - self.prependEnvVar("BISON_PKGDATADIR", os.path.join(CraftStandardDirs.craftRoot(), "share", "bison")) - self.prependEnvVar("M4", os.path.join(CraftStandardDirs.craftRoot(), "dev-utils", "bin", "m4")) + self.prependEnvVar("LD_LIBRARY_PATH", [os.path.join(CraftCore.standardDirs.craftRoot(), "lib"), + os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "x86_64-linux-gnu")]) + self.prependEnvVar("BISON_PKGDATADIR", os.path.join(CraftCore.standardDirs.craftRoot(), "share", "bison")) + self.prependEnvVar("M4", os.path.join(CraftCore.standardDirs.craftRoot(), "dev-utils", "bin", "m4")) def _setupWin(self): if not "HOME" in os.environ: self.addEnvVar("HOME", os.getenv("USERPROFILE")) if CraftCore.compiler.isMinGW(): if not CraftCore.settings.getboolean("QtSDK", "Enabled", "false"): if CraftCore.compiler.isX86(): - self.prependEnvVar("PATH", os.path.join(CraftStandardDirs.craftRoot(), "mingw", "bin")) + self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "mingw", "bin")) else: - self.prependEnvVar("PATH", os.path.join(CraftStandardDirs.craftRoot(), "mingw64", "bin")) + self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "mingw64", "bin")) else: compilerName = CraftCore.settings.get("QtSDK", "Compiler") compilerMap = {"mingw53_32": "mingw530_32"} self.prependEnvVar("PATH", os.path.join(CraftCore.settings.get("QtSDK", "Path"), "Tools", compilerMap.get(compilerName, compilerName), "bin")) if CraftCore.settings.getboolean("QtSDK", "Enabled", "false"): self.prependEnvVar("PATH", os.path.join(CraftCore.settings.get("QtSDK", "Path"), CraftCore.settings.get("QtSDK", "Version"), CraftCore.settings.get("QtSDK", "Compiler"), "bin")) if CraftCore.compiler.isMinGW(): if not CraftCore.settings.getboolean("QtSDK", "Enabled", "false"): if CraftCore.compiler.isX86(): - self.prependEnvVar("PATH", os.path.join(CraftStandardDirs.craftRoot(), "mingw", "bin")) + self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "mingw", "bin")) else: - self.prependEnvVar("PATH", os.path.join(CraftStandardDirs.craftRoot(), "mingw64", "bin")) + self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "mingw64", "bin")) else: compilerName = CraftCore.settings.get("QtSDK", "Compiler") compilerMap = {"mingw53_32": "mingw530_32"} self.prependEnvVar("PATH", os.path.join(CraftCore.settings.get("QtSDK", "Path"), "Tools", compilerMap.get(compilerName, compilerName), "bin")) def setupEnvironment(self): originaleEnv = CaseInsensitiveDict(os.environ) for var, value in CraftCore.settings.getSection("Environment"): # set and override existing values # the ini is case insensitive so sections are lowercase.... self.addEnvVar(var.upper(), value) self.prependEnvVar("PATH", os.path.dirname(sys.executable)) os.environ = self.getEnv() - self.checkForEvilApplication() - self.addEnvVar("KDEROOT", CraftStandardDirs.craftRoot()) + self.addEnvVar("KDEROOT", CraftCore.standardDirs.craftRoot()) self.addEnvVar("SSL_CERT_FILE", os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")) self.addEnvVar("REQUESTS_CA_BUNDLE", os.path.join(CraftCore.standardDirs.etcDir(), "cacert.pem")) if CraftCore.settings.getboolean("Compile", "UseCCache", False): self.addEnvVar("CCACHE_DIR", - CraftCore.settings.get("Paths", "CCACHE_DIR", os.path.join(CraftStandardDirs.craftRoot(), + CraftCore.settings.get("Paths", "CCACHE_DIR", os.path.join(CraftCore.standardDirs.craftRoot(), "build", "CCACHE"))) if CraftCore.settings.getboolean("QtSDK", "Enabled", "false"): sdkPath = os.path.join(CraftCore.settings.get("QtSDK", "Path"), CraftCore.settings.get("QtSDK", "Version"), CraftCore.settings.get("QtSDK", "Compiler"), "bin") if not os.path.exists(sdkPath): log(f"Please ensure that you have installed the Qt SDK in {sdkPath}", critical=True) self.prependEnvVar("PATH", sdkPath) if OsUtils.isWin(): self._setupWin() else: self._setupUnix() - PKG_CONFIG_PATH = collections.OrderedDict.fromkeys([os.path.join(CraftStandardDirs.craftRoot(), "lib", "pkgconfig")]) + PKG_CONFIG_PATH = collections.OrderedDict.fromkeys([os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "pkgconfig")]) if "PKG_CONFIG_PATH" in originaleEnv: PKG_CONFIG_PATH.update(collections.OrderedDict.fromkeys(originaleEnv["PKG_CONFIG_PATH"].split(os.path.pathsep))) else: pkgCOnfig = shutil.which("pkg-config", path=originaleEnv["PATH"]) if pkgCOnfig: out = self._getOutput("pkg-config --variable pc_path pkg-config", shell=True) if out[0] == 0: PKG_CONFIG_PATH.update(collections.OrderedDict.fromkeys(out[1].split(os.path.pathsep))) self.prependEnvVar("PKG_CONFIG_PATH", os.path.pathsep.join(PKG_CONFIG_PATH.keys())) - self.prependEnvVar("QT_PLUGIN_PATH", [os.path.join(CraftStandardDirs.craftRoot(), "plugins"), - os.path.join(CraftStandardDirs.craftRoot(), "lib", "plugins"), - os.path.join(CraftStandardDirs.craftRoot(), "lib64", "plugins"), - os.path.join(CraftStandardDirs.craftRoot(), "lib", "x86_64-linux-gnu", + self.prependEnvVar("QT_PLUGIN_PATH", [os.path.join(CraftCore.standardDirs.craftRoot(), "plugins"), + os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "plugins"), + os.path.join(CraftCore.standardDirs.craftRoot(), "lib64", "plugins"), + os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "x86_64-linux-gnu", "plugins"), - os.path.join(CraftStandardDirs.craftRoot(), "lib", "plugin") + os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "plugin") ]) - self.prependEnvVar("QML2_IMPORT_PATH", [os.path.join(CraftStandardDirs.craftRoot(), "qml"), - os.path.join(CraftStandardDirs.craftRoot(), "lib", "qml"), - os.path.join(CraftStandardDirs.craftRoot(), "lib64", "qml"), - os.path.join(CraftStandardDirs.craftRoot(), "lib", "x86_64-linux-gnu", + self.prependEnvVar("QML2_IMPORT_PATH", [os.path.join(CraftCore.standardDirs.craftRoot(), "qml"), + os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "qml"), + os.path.join(CraftCore.standardDirs.craftRoot(), "lib64", "qml"), + os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "x86_64-linux-gnu", "qml") ]) self.prependEnvVar("QML_IMPORT_PATH", os.environ["QML2_IMPORT_PATH"]) self.prependEnvVar("QT_DATA_DIRS", CraftCore.standardDirs.locations.data) self.setXDG() - self.prependEnvVar("PATH", CraftStandardDirs.craftBin()) + self.prependEnvVar("PATH", CraftCore.standardDirs.craftBin()) # make sure thate craftroot bin is the first to look for dlls etc - self.prependEnvVar("PATH", os.path.join(CraftStandardDirs.craftRoot(), "bin")) - self.prependEnvVar("PATH", os.path.join(CraftStandardDirs.craftRoot(), "dev-utils", "bin")) + self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "bin")) + self.prependEnvVar("PATH", os.path.join(CraftCore.standardDirs.craftRoot(), "dev-utils", "bin")) # add python site packages to pythonpath - self.prependEnvVar("PYTHONPATH", os.path.join(CraftStandardDirs.craftRoot(), "lib", "site-packages")) + self.prependEnvVar("PYTHONPATH", os.path.join(CraftCore.standardDirs.craftRoot(), "lib", "site-packages")) if CraftCore.compiler.isClang(): if OsUtils.isUnix(): self.addEnvVar("CC", "/usr/bin/clang") self.addEnvVar("CXX", "/usr/bin/clang++") else: if CraftCore.compiler.isMSVC(): self.addEnvVar("CC", "clang-cl") self.addEnvVar("CXX", "clang-cl") else: self.addEnvVar("CC", "clang") self.addEnvVar("CXX", "clang") elif CraftCore.compiler.isGCC(): if not CraftCore.compiler.isNative() and CraftCore.compiler.isX86(): self.addEnvVar("CC", "gcc -m32") self.addEnvVar("CXX", "g++ -m32") self.addEnvVar("AS", "gcc -c -m32") else: self.addEnvVar("CC", "gcc") self.addEnvVar("CXX", "g++") if CraftCore.settings.getboolean("General", "AllowAnsiColor", False): self.addEnvVar("CLICOLOR_FORCE", "1") self.addEnvVar("CLICOLOR", "1") if CraftCore.compiler.isClang() and CraftCore.compiler.isMSVC(): self.prependEnvVar("CFLAGS", "-fcolor-diagnostics -fansi-escape-codes", sep=" ") self.prependEnvVar("CXXFLAGS", "-fcolor-diagnostics -fansi-escape-codes", sep=" ") elif CraftCore.compiler.isGCCLike(): self.prependEnvVar("CFLAGS", "-fdiagnostics-color=always", sep=" ") self.prependEnvVar("CXXFLAGS", "-fdiagnostics-color=always", sep=" ") if OsUtils.isWin(): os.environ["TERM"] = "xterm" # pretend to be a common smart terminal def printEnv(self): self.setupEnvironment() for key, val in os.environ.items(): if "\n" in val: log(f"Not adding ${key} to environment since it contains " "a newline character and that breaks craftenv.sh") continue if key.startswith("BASH_FUNC_"): continue # weird protected env vars if key in {"PROFILEREAD"}: continue CraftCore.log.info(f"{key}={val}") @property def version(self): return CraftCore.settings.version +helper = SetupHelper() if __name__ == '__main__': - helper = SetupHelper() helper.run() + diff --git a/bin/Utils/CraftShortPath.py b/bin/Utils/CraftShortPath.py index 7d74f8d62..a1690d994 100644 --- a/bin/Utils/CraftShortPath.py +++ b/bin/Utils/CraftShortPath.py @@ -1,68 +1,66 @@ import os import zlib import subprocess from CraftCore import CraftCore -import CraftConfig -import CraftDebug from CraftOS.osutils import OsUtils class CraftShortPath(object): _useShortpaths = OsUtils.isWin() _shortPaths = {} def __init__(self, path, createShortPath=None) -> None: self._longPath = path self._shortPath = None if not createShortPath: self._createShortPathLambda = CraftShortPath._createShortPath else: self._createShortPathLambda = createShortPath def path(self, condition): return self.shortPath if condition else self.longPath @property def longPath(self) -> str: return self._longPath() if callable(self._longPath) else self._longPath @property def shortPath(self) -> str: if self._shortPath: return self._shortPath self._shortPath = CraftShortPath._shortPaths.get(self.longPath, None) if not self._shortPath: self._shortPath = self._createShortPathLambda(self.longPath) CraftShortPath._shortPaths[self.longPath] = self._shortPath if self._shortPath != self.longPath: os.makedirs(self.longPath, exist_ok=True) CraftCore.debug.log.debug(f"Mapped \n" f"{self.longPath} to\n" f"{self._shortPath}, gained {len(self.longPath) - len(self._shortPath)}") return self._shortPath @staticmethod def _createShortPath(longPath) -> str: import utils longPath = OsUtils.toNativePath(longPath) if not CraftShortPath._useShortpaths: return longPath if not os.path.isdir(CraftCore.standardDirs.junctionsDir()): os.makedirs(CraftCore.standardDirs.junctionsDir()) path = OsUtils.toNativePath(os.path.join(CraftCore.standardDirs.junctionsDir(), hex(zlib.crc32(bytes(longPath, "UTF-8")))[2:])) if len(longPath) < len(path): CraftCore.debug.log.debug(f"Using junctions for {longPath} wouldn't save characters returning original path") CraftCore.debug.log.debug(f"{longPath}\n" f"{path}, gain:{len(longPath) - len(path)}") return longPath os.makedirs(longPath, exist_ok=True) if not os.path.isdir(path): # note: mklink is a CMD command => needs shell if not utils.system(["mklink", "/J", path, longPath], shell=True, stdout=subprocess.DEVNULL, logCommand=False): CraftCore.debug.log.critical(f"Could not create shortpath {path}, for {longPath}") return longPath else: if not os.path.samefile(path, longPath): CraftCore.debug.log.critical(f"Existing short path {path}, did not match {longPath}") return longPath return path diff --git a/bin/craft.py b/bin/craft.py index 335e70f31..f39a79d51 100755 --- a/bin/craft.py +++ b/bin/craft.py @@ -1,267 +1,264 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright Holger Schroeder # Copyright Patrick Spendrin # Copyright Hannah von Reth # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. import argparse import collections import sys import CraftCommands import CraftSetupHelper import InstallDB import blueprintSearch from Blueprints.CraftPackageObject import * from CraftCore import CraftCore from Utils import CraftTimer from Utils.CraftTitleUpdater import CraftTitleUpdater from options import UserOptions class ActionHandler: class StoreTrueAction(argparse._StoreTrueAction): def __call__(self, parser, namespace, values, option_string=None): ActionHandler.StoreAction._addOrdered(namespace, self.dest, self.const) super().__call__(parser, namespace, values, option_string) class StoreAction(argparse._StoreAction): """http://stackoverflow.com/a/9028031""" def __call__(self, parser, namespace, values, option_string=None): ActionHandler.StoreAction._addOrdered(namespace, self.dest, values) super().__call__(parser, namespace, values, option_string) @staticmethod def _addOrdered(namespace, key, value): if not 'ordered_args' in namespace: setattr(namespace, 'ordered_args', collections.OrderedDict()) namespace.ordered_args[key] = value def __init__(self, parser): self.parser = parser self.actions = {} def _addAction(self, actionName, help=None, **kwargs): arg = self.parser.add_argument("--%s" % actionName, help="[Action] %s" % (help if help else ""), **kwargs) self.actions[arg.dest] = actionName def addAction(self, actionName, **kwargs): self._addAction(actionName, action=ActionHandler.StoreTrueAction, **kwargs) def addActionWithArg(self, actionName, **kwargs): self._addAction(actionName, action=ActionHandler.StoreAction, **kwargs) def parseFinalAction(self, args, defaultAction): '''Returns the list of actions or [defaultAction]''' return [self.actions[x] for x in args.ordered_args.keys()] if hasattr(args, "ordered_args") else [defaultAction] def main(): parser = argparse.ArgumentParser(prog="Craft", description="Craft is an open source meta build system and package manager." "It manages dependencies and builds libraries and applications from source, on Windows, Mac, Linux and FreeBSD.", epilog="For more information visit https://community.kde.org/Craft.\n" "Send feedback to .") parser.add_argument("-p", "--probe", action="store_true", help="probing: craft will only look which files it has to build according to the list of installed files and according to the dependencies of the package.") parser.add_argument("--list-file", action="store", help="Build all packages from the ini file provided") parser.add_argument("--options", action="append", default=CraftCore.settings.getList("General", "Options", ""), help="Set craft property from string . An example for is extragear/kdevelop.version=5.3 or [Compile]MakeProgram=jom.") parser.add_argument("-q", "--stayquiet", action="store_true", dest="stayQuiet", help="quiet: there should be no output - The verbose level should be 0") parser.add_argument("--create-cache", action="store_true", dest="createCache", default=CraftCore.settings.getboolean("Packager", "CreateCache", "False"), help="Create a binary cache, the setting is overwritten by --no-cache") parser.add_argument("--use-cache", action="store_true", dest="useCache", default=CraftCore.settings.getboolean("Packager", "UseCache", "False"), help="Use a binary cache, the setting is overwritten by --no-cache") parser.add_argument("--no-cache", action="store_true", dest="noCache", default=False, help="Don't create or use the binary cache") parser.add_argument("--destroy-craft-root", action="store_true", dest="doDestroyCraftRoot", default=False, help="DANGEROUS: Recursively delete everything in the Craft root directory besides the CraftSettings.ini, the download directory and the craft folder itself") parser.add_argument("--offline", action="store_true", default=CraftCore.settings.getboolean("General", "WorkOffline", False), help="do not try to connect to the internet: KDE packages will try to use an existing source tree and other packages would try to use existing packages in the download directory.\ If that doesn't work, the build will fail.") parser.add_argument("--buildtype", choices=["Release", "RelWithDebInfo", "MinSizeRel", "Debug"], dest="buildType", default=CraftCore.settings.get("Compile", "BuildType", "RelWithDebInfo"), help="This will override the build type set in your CraftSettings.ini.") parser.add_argument("-v", "--verbose", action="count", default=int(CraftCore.settings.get("CraftDebug", "Verbose", "0")), help=" verbose: increases the verbose level of craft. Default is 1. verbose level 1 contains some notes from craft, all output of cmake, make and other programs that are used.\ verbose level 2a dds an option VERBOSE=1 to make and craft is more verbose highest level is verbose level 3.") parser.add_argument("-i", "--ignoreInstalled", action="store_true", help="ignore install: using this option will install a package over an existing install. This can be useful if you want to check some new code and your last build isn't that old.") parser.add_argument("--resolve-deps", action="store", help="Similar to -i, all dependencies will be resolved and the action is applied on them") parser.add_argument("--target", action="store", help="This will override the build of the default target.") parser.add_argument("--search", action="store_true", help="This will search for a package or a description matching or similar to the search term.") parser.add_argument("--src-dir", action="store", dest="srcDir", help="This will override the source dir and enable the offline mode") parser.add_argument("--ci-mode", action="store_true", default=CraftCore.settings.getboolean("ContinuousIntegration", "Enabled", False), dest="ciMode", help="Enables the ci mode") parser.add_argument("--add-blueprint-repository", action="store", help="Installs a blueprint repository", metavar="URL") actionHandler = ActionHandler(parser) for x in sorted(["fetch", "fetch-binary", "unpack", "configure", ("compile",{"help":"Same as --configure --make"}), "make", "install", "install-deps", "qmerge", "post-qmerge", "post-install", "package", "unmerge", "test", "createpatch"], key=lambda x: x[0] if isinstance(x, tuple) else x): if isinstance(x, tuple): actionHandler.addAction(x[0], **x[1]) else: actionHandler.addAction(x) actionHandler.addAction("update", help="Update all installed packages") # read-only actions actionHandler.addAction("print-installed", help="This will show a list of all packages that are installed currently.") actionHandler.addAction("print-files", help="Print the files installed by the package and exit") actionHandler.addActionWithArg("search-file", help="Print packages owning the file") actionHandler.addActionWithArg("get", help="Get any value from a Blueprint") actionHandler.addActionWithArg("set", help="Permanently set a config value of a Blueprint") actionHandler.addActionWithArg("run", nargs="+", help="Run an application in the Craft environment") actionHandler.addAction("clean-unused", help="Clean unused files of all packages") # other actions parser.add_argument("--version", action="version", version = f"%(prog)s {CraftSetupHelper.SetupHelper.CraftVersion}") parser.add_argument("packageNames", nargs=argparse.REMAINDER) args = parser.parse_args() if args.stayQuiet: CraftCore.debug.setVerbose(-1) elif args.verbose: CraftCore.debug.setVerbose(args.verbose) CraftCore.settings.set("General", "WorkOffline", args.offline or args.srcDir is not None) CraftCore.settings.set("Compile", "BuildType", args.buildType) CraftCore.settings.set("General", "Options", ";".join(args.options)) CraftCore.settings.set("Packager", "CreateCache", not args.noCache and args.createCache) CraftCore.settings.set("Packager", "UseCache", not args.noCache and args.useCache) CraftCore.settings.set("ContinuousIntegration", "SourceDir", args.srcDir) CraftCore.settings.set("ContinuousIntegration", "Enabled", args.ciMode) - helper = CraftSetupHelper.SetupHelper() - if not "KDEROOT" in os.environ: - helper.setupEnvironment() - helper.printBanner() + CraftSetupHelper.SetupHelper.printBanner() if args.doDestroyCraftRoot: return CraftCommands.destroyCraftRoot() if args.run: return utils.system(args.run, shell=True) if args.add_blueprint_repository: return CraftCommands.addBlueprintsRepository(args.add_blueprint_repository, args) if CraftCore.settings.getboolean("Packager", "CreateCache"): # we are in cache creation mode, ensure to create a 7z image and not an installer CraftCore.settings.set("Packager", "PackageType", "SevenZipPackager") UserOptions.setOptions(args.options) if args.search: for package in args.packageNames: blueprintSearch.printSearch(package) return True for action in actionHandler.parseFinalAction(args, "all"): tempArgs = copy.deepcopy(args) if action in ["install-deps", "package"]: tempArgs.ignoreInstalled = True CraftCore.log.debug("buildAction: %s" % action) CraftCore.log.debug("doPretend: %s" % tempArgs.probe) CraftCore.log.debug("packageName: %s" % tempArgs.packageNames) CraftCore.log.debug("buildType: %s" % tempArgs.buildType) CraftCore.log.debug("verbose: %d" % CraftCore.debug.verbose()) CraftCore.log.debug("Craft: %s" % CraftCore.standardDirs.craftRoot()) packageNames = tempArgs.packageNames if tempArgs.list_file: if not os.path.exists(tempArgs.list_file): CraftCore.log.error(f"List file {tempArgs.list_file!r} does not exist") return False if not packageNames: packageNames = [] packageNames += CraftCommands.readListFile(tempArgs.list_file) if action == "print-installed": InstallDB.printInstalled() elif action == "search-file": InstallDB.printPackagesForFileSearch(tempArgs.search_file) elif action == "set": CraftCommands.setOption(packageNames, args.set) elif action == "clean-unused": CraftCommands.cleanBuildFiles(cleanArchives=True, cleanImages=True, cleanInstalledImages=False, cleanBuildDir=True, packages=blueprintSearch.packages()) elif action == "update": return CraftCommands.updateInstalled(tempArgs) else: if not packageNames: return True package = CraftCommands.resolvePackage(packageNames, version=tempArgs.target) if not CraftCommands.run(package, action, tempArgs): return False return True if __name__ == '__main__': success = False with CraftTimer.Timer("Craft", 0) as timer: CraftTitleUpdater.instance = CraftTitleUpdater() if not "CRAFT_NOTITLEUPDATE" in os.environ and not "--ci-mode" in sys.argv: CraftTitleUpdater.instance.start(f"({CraftCore.standardDirs.craftRoot()}) craft " + " ".join(sys.argv[1:]), timer) try: success = main() except KeyboardInterrupt: pass except BlueprintNotFoundException as e: CraftCore.log.error(e) blueprintSearch.printSearch(e.packageName) except BlueprintException as e: if 0 <= CraftCore.debug.verbose() < 2: CraftCore.log.error(e) CraftCore.log.debug(e, exc_info=e.exception or e) else: CraftCore.log.error(e, exc_info=e.exception or e) except Exception as e: CraftCore.log.error(e, exc_info=e) finally: CraftTitleUpdater.instance.stop() if not success: exit(1)