diff --git a/bin/CraftBase.py b/bin/CraftBase.py index 0410abd9a..5c9e58c52 100644 --- a/bin/CraftBase.py +++ b/bin/CraftBase.py @@ -1,271 +1,280 @@ # # copyright (c) 2009 Ralf Habacker # import datetime import functools import sys import utils from CraftConfig import * from CraftCore import CraftCore from CraftStandardDirs import CraftStandardDirs from Blueprints import CraftPackageObject from CraftDebug import deprecated from Blueprints.CraftPackageObject import CraftPackageObject from Utils.CraftShortPath import CraftShortPath class InitGuard(object): _initialized = {} _verbose = False @staticmethod def _dummy_init(key, *args, **kwargs): if InitGuard._verbose: print("dummy_init", key) args[0].__dict__ = InitGuard._initialized[key].__dict__ @staticmethod def init_once(fun): @functools.wraps(fun) def inner(*args, **kwargs): if fun.__name__ != "__init__": raise Exception("InitGuard can only handle __init__ calls") key = (args[0].__class__, fun.__code__) if key not in InitGuard._initialized: if InitGuard._verbose: print("init", key) InitGuard._initialized[key] = args[0] return fun(*args, **kwargs) else: return InitGuard._dummy_init(key, *args, **kwargs) return inner class CraftBase(object): """base class for craft system - holds attributes and methods required by base classes""" @InitGuard.init_once def __init__(self): # TODO: some __init__ of subclasses need to already have been # called here. That is really the wrong way round. object.__init__(self) CraftCore.log.debug("CraftBase.__init__ called") mod = sys.modules[self.__module__] # ugly workaround we need to replace the constructor self.package = mod.CRAFT_CURRENT_MODULE # type: CraftPackageObject self.subinfo = mod.subinfo(self) self.buildSystemType = None def __str__(self): return self.package.__str__() @property def noFetch(self): return CraftCore.settings.getboolean("General", "WorkOffline", False) @property def buildTests(self): if self.subinfo.options.dynamic.buildTests is not None: return self.subinfo.options.dynamic.buildTests # TODO: remove deprecated "Compile", "BuildTests" and provide a default for buildTests if ("Compile", "BuildTests") in CraftCore.settings: CraftCore.debug.print(f"[Compile]BuildTests is deprecated please use [{self}]buildTests instead", file=sys.stderr) return CraftCore.settings.getboolean("Compile", "BuildTests", True) def buildType(self): """return currently selected build type""" return CraftCore.settings.get("Compile", "BuildType") def buildArchitecture(self): """return the target CPU architecture""" CraftCore.compiler.architecture() def workDirPattern(self): """return base directory name for package related work directory""" return f"{self.buildType()}-{self.buildTarget}" def imageDirPattern(self): """return base directory name for package related image directory""" return f"image-{self.buildType()}-{self.buildTarget}" def sourceDir(self, dummyIndex=0): utils.abstract() def packageDir(self): """ add documentation """ return os.path.dirname(self.package.source) def installPrefix(self): prefix = CraftCore.standardDirs.craftRoot() if self.buildTarget in self.subinfo.targetInstallPath: prefix = os.path.join(prefix, self.subinfo.targetInstallPath[self.buildTarget]) return prefix def buildRoot(self): """return absolute path to the root directory of the currently active package""" return os.path.realpath(os.path.join(CraftStandardDirs.craftRoot(), "build", self.package.path)) def workDir(self): """return absolute path to the 'work' subdirectory of the currently active package""" work = os.path.join(self.buildRoot(), "work") return CraftShortPath(work).shortPath def buildDir(self): if not self.subinfo.options.useShadowBuild: return self.sourceDir() CraftCore.log.debug("CraftBase.buildDir() called") builddir = os.path.join(self.workDir(), self.workDirPattern()) CraftCore.log.debug(f"package builddir is: {builddir}") return builddir def imageDir(self): """return absolute path to the install root directory of the currently active package """ return os.path.join(self.buildRoot(), self.imageDirPattern()) def installDir(self): """return absolute path to the install directory of the currently active package. This path may point to a subdir of imageDir() in case @ref info.targetInstallPath is used """ if self.subinfo.hasInstallPath(): installDir = os.path.join(self.imageDir(), self.subinfo.installPath()) else: installDir = self.imageDir() return installDir def packageDestinationDir(self, withBuildType=True): """return absolute path to the directory where binary packages are placed into. Default is to optionally append build type subdirectory""" CraftCore.log.debug("CraftBase.packageDestinationDir called") dstpath = CraftCore.settings.get("Packager", "Destination", os.path.join(CraftStandardDirs.craftRoot(), "tmp")) if not os.path.exists(dstpath): utils.createDir(dstpath) return dstpath @property def buildTarget(self): return self.subinfo.buildTarget @property def version(self): ver = self.subinfo.buildTarget if CraftCore.settings.getboolean("BlueprintVersions", "EnableDailyUpdates", True)\ and self.subinfo.options.dailyUpdate and self.subinfo.hasSvnTarget(): ver += "-" + str(datetime.date.today()).replace("-", ".") elif self.subinfo.options.dynamic.patchLevel: ver += f"-{self.subinfo.options.dynamic.patchLevel}" elif self.subinfo.buildTarget in self.subinfo.patchLevel: ver += f"-{self.subinfo.patchLevel[self.subinfo.buildTarget]}" return ver @property def rootdir(self): return CraftStandardDirs.craftRoot() def enterBuildDir(self): CraftCore.debug.trace("CraftBase.enterBuildDir called") utils.createDir(self.buildDir()) os.chdir(self.buildDir()) CraftCore.log.debug("entering: %s" % self.buildDir()) def enterSourceDir(self): if (not os.path.exists(self.sourceDir())): return False CraftCore.log.warning("entering the source directory!") os.chdir(self.sourceDir()) CraftCore.log.debug("entering: %s" % self.sourceDir()) def buildNumber(self): if "APPVEYOR_BUILD_VERSION" in os.environ: return os.environ["APPVEYOR_BUILD_VERSION"] elif "BUILD_NUMBER" in os.environ: return os.environ["BUILD_NUMBER"] return "" def binaryArchiveName(self, pkgSuffix=None, fileType=CraftCore.settings.get("Packager", "7ZipArchiveType", "7z"), includeRevision=False, includePackagePath=False, includeTimeStamp=False) -> str: if not pkgSuffix: pkgSuffix = "" if hasattr(self.subinfo.options.package, 'packageSuffix') and self.subinfo.options.package.packageSuffix: pkgSuffix = self.subinfo.options.package.packageSuffix buildVersion = self.buildNumber() version = [] if not buildVersion and self.subinfo.hasSvnTarget(): if includeRevision: version += [self.sourceRevision(), buildVersion] else: version += ["latest"] else: version = [self.version, buildVersion] if includeTimeStamp: version += [datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%S")] version = "-".join(filter(None, version)) version = version.replace("/", "_") if fileType: fileType = f".{fileType}" else: fileType = "" prefix = "" if not includePackagePath else f"{self.package.path}/" return f"{prefix}{self.package.name}-{version}-{CraftCore.compiler}{pkgSuffix}{fileType}" @staticmethod def cacheVersion(): if CraftCore.settings.getboolean("QtSDK", "Enabled", "False"): version = CraftCore.settings.get("QtSDK", "Version") return f"QtSDK_{version}" else: return CraftCore.settings.get("Packager", "CacheVersion") def cacheLocation(self, baseDir=None) -> str: if not baseDir: cacheDir = CraftCore.settings.get("Packager", "CacheDir", os.path.join(CraftStandardDirs.downloadDir(), "binary")) else: cacheDir = baseDir version = self.cacheVersion() if not version: return None return os.path.join(cacheDir, version, *CraftCore.compiler.signature, self.buildType()) def cacheRepositoryUrls(self) -> [str]: version = self.cacheVersion() buildType = [self.buildType()] if self.buildType() == "RelWithDebInfo": buildType += ["Release"] elif self.buildType() == "Release": buildType += ["RelWithDebInfo"] out = [] for bt in buildType: out += ["/".join([url if not url.endswith("/") else url[0:-1], version, *CraftCore.compiler.signature, bt]) for url in CraftCore.settings.getList("Packager", "RepositoryUrl")] return out def internalPostInstall(self): return True def postInstall(self): return True def internalPostQmerge(self): return True def postQmerge(self): return True def cleanImage(self) -> bool: """cleanup before install to imagedir""" if (os.path.exists(self.imageDir())): CraftCore.log.debug("cleaning image dir: %s" % self.imageDir()) utils.cleanDirectory(self.imageDir()) os.rmdir(self.imageDir()) return True + + def cleanBuild(self) -> bool: + """cleanup currently used build dir""" + if not self.subinfo.options.useShadowBuild: + return True + if os.path.exists(self.buildDir()): + utils.cleanDirectory(self.buildDir()) + CraftCore.log.debug("cleaning build dir: %s" % self.buildDir()) + return True diff --git a/bin/Package/PackageBase.py b/bin/Package/PackageBase.py index c0d3f6177..04a36c16c 100644 --- a/bin/Package/PackageBase.py +++ b/bin/Package/PackageBase.py @@ -1,270 +1,261 @@ # # copyright (c) 2009 Ralf Habacker # from CraftBase import * from CraftCompiler import * from InstallDB import * from Blueprints.CraftPackageObject import * from Utils import CraftHash, GetFiles, CraftChoicePrompt from Utils.CraftManifest import CraftManifest import json class PackageBase(CraftBase): """ provides a generic interface for packages and implements the basic stuff for all packages """ # uses the following instance variables # todo: place in related ...Base # rootdir -> CraftBase # package -> PackageBase # force -> PackageBase # category -> PackageBase # version -> PackageBase # packagedir -> PackageBase # imagedir -> PackageBase def __init__(self): CraftCore.log.debug("PackageBase.__init__ called") CraftBase.__init__(self) def qmerge(self): """mergeing the imagedirectory into the filesystem""" ## \todo is this the optimal place for creating the post install scripts ? if self.package.isInstalled: self.unmerge() copiedFiles = [] # will be populated by the next call if not utils.copyDir(self.imageDir(), CraftCore.standardDirs.craftRoot(), copiedFiles=copiedFiles): return False # add package to installed database -> is this not the task of the manifest files ? revision = self.sourceRevision() package = CraftCore.installdb.addInstalled(self.package, self.version, revision=revision) fileList = self.getFileListFromDirectory(CraftCore.standardDirs.craftRoot(), copiedFiles) package.addFiles(fileList) package.install() if (CraftCore.settings.getboolean("Packager", "CreateCache") or CraftCore.settings.getboolean("Packager", "UseCache")): package.setCacheVersion(self.cacheVersion()) return True def unmerge(self): """unmergeing the files from the filesystem""" CraftCore.log.debug("Packagebase unmerge called") packageList = CraftCore.installdb.getInstalledPackages(self.package) for package in packageList: fileList = package.getFilesWithHashes() self.unmergeFileList(CraftCore.standardDirs.craftRoot(), fileList) package.uninstall() return True - def cleanBuild(self) -> bool: - """cleanup currently used build dir""" - if not self.subinfo.options.useShadowBuild: - return True - if os.path.exists(self.buildDir()): - utils.cleanDirectory(self.buildDir()) - CraftCore.log.debug("cleaning build dir: %s" % self.buildDir()) - return True - def strip(self, fileName): """strip debugging informations from shared libraries and executables - mingw only!!! """ if self.subinfo.options.package.disableStriping or CraftCore.compiler.isMSVC() or not CraftCore.compiler.isGCCLike(): CraftCore.log.warning(f"Skipping stripping of {fileName} -- either disabled or unsupported with this compiler") return True if OsUtils.isMac(): CraftCore.log.debug(f"Skipping stripping of files on macOS -- not implemented") return True if os.path.isabs(fileName): filepath = fileName else: CraftCore.log.warning("Please pass an absolute file path to strip") basepath = os.path.join(self.installDir()) filepath = os.path.join(basepath, "bin", fileName) return utils.system(["strip", "-s", filepath]) def createImportLibs(self, pkgName): """create the import libraries for the other compiler(if ANSI-C libs)""" basepath = os.path.join(self.installDir()) utils.createImportLibs(pkgName, basepath) def printFiles(self): packageList = CraftCore.installdb.getInstalledPackages(self.package) for package in packageList: fileList = package.getFiles() fileList.sort() for file in fileList: print(file[0]) return True def getAction(self, cmd=None): if not cmd: command = sys.argv[1] options = None # print sys.argv if (len(sys.argv) > 2): options = sys.argv[2:] else: command = cmd options = None # \todo options are not passed through by craft.py fix it return [command, options] def execute(self, cmd=None): """called to run the derived class this will be executed from the package if the package is started on its own it shouldn't be called if the package is imported as a python module""" CraftCore.log.debug("PackageBase.execute called. args: %s" % sys.argv) command, _ = self.getAction(cmd) return self.runAction(command) def fetchBinary(self, downloadRetriesLeft=3) -> bool: if self.subinfo.options.package.disableBinaryCache: return False for url in [self.cacheLocation()] + self.cacheRepositoryUrls(): CraftCore.log.debug(f"Trying to restore {self} from cache: {url}.") if url == self.cacheLocation(): fileUrl = f"{url}/manifest.json" if os.path.exists(fileUrl): with open(fileUrl, "rt", encoding="UTF-8") as f: manifest = CraftManifest.fromJson(json.load(f)) else: continue else: manifest = CraftManifest.fromJson(CraftCore.cache.cacheJsonFromUrl(f"{url}/manifest.json")) fileEntry = manifest.get(str(self)).files files = [] for f in fileEntry: if f.version == self.version: files.append(f) latest = None if not files: CraftCore.log.debug(f"Could not find {self}={self.version} in {url}") continue latest = files[0] if url != self.cacheLocation(): downloadFolder = self.cacheLocation(os.path.join(CraftCore.standardDirs.downloadDir(), "cache")) else: downloadFolder = self.cacheLocation() localArchiveAbsPath = OsUtils.toNativePath(os.path.join(downloadFolder, latest.fileName)) localArchivePath, localArchiveName = os.path.split(localArchiveAbsPath) if url != self.cacheLocation(): if not os.path.exists(localArchiveAbsPath): os.makedirs(localArchivePath, exist_ok=True) fUrl = f"{url}/{latest.fileName}" if not GetFiles.getFile(fUrl, localArchivePath, localArchiveName): CraftCore.log.warning(f"Failed to fetch {fUrl}") return False elif not os.path.isfile(localArchiveAbsPath): continue if not CraftHash.checkFilesDigests(localArchivePath, [localArchiveName], digests=latest.checksum, digestAlgorithm=CraftHash.HashAlgorithm.SHA256): CraftCore.log.warning(f"Hash did not match, {localArchiveName} might be corrupted") if downloadRetriesLeft and CraftChoicePrompt.promptForChoice("Do you want to delete the files and redownload them?", [("Yes", True), ("No", False)], default="Yes"): return utils.deleteFile(localArchiveAbsPath) and self.fetchBinary(downloadRetriesLeft=downloadRetriesLeft-1) return False self.subinfo.buildPrefix = latest.buildPrefix if not (self.cleanImage() and utils.unpackFile(localArchivePath, localArchiveName, self.imageDir()) and self.internalPostInstall() and self.postInstall() and self.qmerge() and self.internalPostQmerge() and self.postQmerge()): return False return True return False @staticmethod def getFileListFromDirectory(imagedir, filePaths): """ create a file list containing hashes """ ret = [] algorithm = CraftHash.HashAlgorithm.SHA256 for filePath in filePaths: relativeFilePath = os.path.relpath(filePath, imagedir) digest = algorithm.stringPrefix() + CraftHash.digestFile(filePath, algorithm) ret.append((relativeFilePath, digest)) return ret @staticmethod def unmergeFileList(rootdir, fileList): """ delete files in the fileList if has matches """ for filename, filehash in fileList: fullPath = os.path.join(rootdir, os.path.normcase(filename)) if os.path.isfile(fullPath) or os.path.islink(fullPath): if filehash: algorithm = CraftHash.HashAlgorithm.getAlgorithmFromPrefix(filehash) currentHash = algorithm.stringPrefix() + CraftHash.digestFile(fullPath, algorithm) if not filehash or currentHash == filehash: OsUtils.rm(fullPath, True) else: CraftCore.log.warning( f"We can't remove {fullPath} as its hash has changed," f" that usually implies that the file was modified or replaced") elif not os.path.isdir(fullPath) and os.path.lexists(fullPath): CraftCore.log.debug(f"Remove a dead symlink {fullPath}") OsUtils.rm(fullPath, True) elif not os.path.isdir(fullPath): CraftCore.log.warning("file %s does not exist" % fullPath) containingDir = os.path.dirname(fullPath) if os.path.exists(containingDir) and not os.listdir(containingDir): CraftCore.log.debug(f"Delete empty dir {containingDir}") utils.rmtree(containingDir) def runAction(self, command): functions = {"fetch": "fetch", "cleanimage": "cleanImage", "cleanbuild": "cleanBuild", "unpack": "unpack", "compile": "compile", "configure": "configure", "make": "make", "install": ["install", "internalPostInstall"], "post-install": "postInstall", "test": "unittest", "qmerge": ["qmerge", "internalPostQmerge"], "post-qmerge": "postQmerge", "unmerge": "unmerge", "package": "createPackage", "createpatch": "createPatch", "print-files": "printFiles", "checkdigest": "checkDigest", "fetch-binary": "fetchBinary"} if command in functions: try: steps = functions[command] if not isinstance(steps, list): steps = [steps] for step in steps: if not getattr(self, step)(): return False except AttributeError as e: raise BlueprintException(str(e), self.package, e) else: CraftCore.log.error("command %s not understood" % command) return False return True diff --git a/bin/Source/VersionSystemSourceBase.py b/bin/Source/VersionSystemSourceBase.py index 52f624fa3..60e25c22f 100644 --- a/bin/Source/VersionSystemSourceBase.py +++ b/bin/Source/VersionSystemSourceBase.py @@ -1,130 +1,130 @@ # # copyright (c) 2009 Ralf Habacker # from Source.SourceBase import * class VersionSystemSourceBase(SourceBase): """abstract base class for version system support""" def __init__(self): CraftCore.debug.trace("VersionSystemSourceBase __init__") SourceBase.__init__(self) def getUrl(self, index): """get the url at position 'index' from a ';' separated list of urls""" CraftCore.debug.trace("VersionSystemSourceBase getUrl") u = self.subinfo.svnTarget() if u.find(';') == -1: if index == 0: return u else: return None # urls are a list urls = u.split(';') if index >= len(urls): return None u = urls[index] return u def splitUrl(self, url): """ split url into real url and url option. the delimiter is '#'""" CraftCore.debug.trace("VersionSystemSourceBase splitUrl") if url.find('#') != -1: return url.split('#') return [url, ""] def __repositoryBaseUrl(self): """ this function return the base url to the KDE repository """ CraftCore.debug.trace("VersionSystemSourceBase __repositoryBaseUrl") # @todo move to SvnSource server = CraftCore.settings.get("General", "KDESVNSERVER", "svn://anonsvn.kde.org") return server + '/home/kde/' def unpack(self): CraftCore.debug.trace("VersionSystemSourceBase unpack") self.enterBuildDir() CraftCore.log.debug("cleaning %s" % self.buildDir()) - utils.cleanDirectory(self.buildDir()) + self.cleanBuild() ret = self.applyPatches() if CraftCore.settings.getboolean("General", "EMERGE_HOLD_ON_PATCH_FAIL", False): return ret return True def repositoryUrlCount(self): """return the number of provided repository url's. Multiple repository urls' are delimited by ';'""" CraftCore.debug.trace("VersionSystemSourceBase repositoryUrlCount") if not self.subinfo.hasSvnTarget(): return 0 u = self.subinfo.svnTarget() if u.find(';') == -1: return 1 urls = u.split(';') return len(urls) def repositoryUrl(self, index=0): """this function returns the full url into a version system based repository at position 'index'. See @ref repositoryUrlCount how to define multiple repository urls.""" CraftCore.debug.trace("VersionSystemSourceBase repositoryUrl") if self.subinfo.hasSvnTarget(): u1 = self.getUrl(index) (u, _) = self.splitUrl(u1) # check relative kde url # @todo this is svn specific - move to SvnSource if u.find("://") == -1 and utils.getVCSType(u) == "svn": url = self.__repositoryBaseUrl() + u else: url = u if url.startswith("["): url = url[url.find("]", 1) + 1:] return url else: return False def repositoryUrlOptions(self, index=0): """this function return options for the repository url at position 'index'. Options for a repository url are defined by adding '#' followed by the specific option. """ CraftCore.debug.trace("VersionSystemSourceBase repositoryUrlOptions") if self.subinfo.hasSvnTarget(): u = self.getUrl(index) (dummy, option) = self.splitUrl(u) return option return None def checkoutDir(self, dummyIndex=0): CraftCore.debug.trace("VersionSystemSourceBase checkoutDir") if ("ContinuousIntegration", "SourceDir") in CraftCore.settings: return CraftCore.settings.get("ContinuousIntegration", "SourceDir") if self.subinfo.hasSvnTarget(): sourcedir = os.path.join(CraftStandardDirs.gitDir(), self.package.path) else: CraftCore.log.critical("svnTarget property not set for this target") if self.subinfo.targetSourceSuffix() != None: sourcedir = "%s-%s" % (sourcedir, self.subinfo.targetSourceSuffix()) return os.path.abspath(sourcedir) def sourceDir(self, index=0): CraftCore.debug.trace("VersionSystemSourceBase sourceDir") if ("ContinuousIntegration", "SourceDir") in CraftCore.settings: return CraftCore.settings.get("ContinuousIntegration", "SourceDir") sourcedir = self.checkoutDir(index) if self.subinfo.hasTargetSourcePath(): sourcedir = os.path.join(sourcedir, self.subinfo.targetSourcePath()) CraftCore.log.debug("using sourcedir: %s" % sourcedir) return os.path.abspath(sourcedir) def sourceRevision(self): CraftCore.debug.trace("VersionSystemSourceBase sourceRevision") if not os.path.exists(self.sourceDir()): # as we are using the cahce we don't have the git clone present return "latest" return self.sourceVersion()