diff --git a/helpers/build-product-dependencies.py b/helpers/build-product-dependencies.py index fd476e3..e478601 100644 --- a/helpers/build-product-dependencies.py +++ b/helpers/build-product-dependencies.py @@ -1,252 +1,254 @@ #!/usr/bin/python3 import os import sys import argparse import subprocess import collections from helperslib import CommonUtils, Buildable # Parse the command line arguments we've been given parser = argparse.ArgumentParser(description='Utility build all dependencies of a given product.') parser.add_argument('--product', type=str, required=True) parser.add_argument('--branchGroup', type=str, required=True) parser.add_argument('--platform', type=str, required=True) parser.add_argument('--environment', type=str, required=True) parser.add_argument('--installTo', type=str, required=True) arguments = parser.parse_args() # Initialise the Dependency Resolver dependencyResolver = Buildable.DependencyResolver() # Ask the Dependency Resolver to load the list of projects... projectsTreeLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'repo-metadata', 'projects' ) dependencyResolver.loadProjectsFromTree( projectsTreeLocation ) # Now ask it to load the list of projects to ignore ignoreFileLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'build-script-ignore' ) dependencyResolver.loadProjectsIgnoreList( ignoreFileLocation ) # As well as our local ignore file ignoreFileLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'ignored-projects' ) dependencyResolver.loadProjectsIgnoreList( ignoreFileLocation ) # Make sure the Platform specific ignore rules are loaded as well ignoreRulesLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'project-ignore-rules.yaml' ) dependencyResolver.loadProjectsIgnoreRules( ignoreRulesLocation ) # Now get it to load the list of dependencies - we have first the common file... dependenciesFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'dependency-data-common' ) dependencyResolver.loadDependenciesFromFile( dependenciesFile ) # And then the branch group specific file... dependenciesFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'dependency-data-' + arguments.branchGroup ) dependencyResolver.loadDependenciesFromFile( dependenciesFile ) # If it exists, we should load an OS specific file as well dependenciesFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'dependency-' + sys.platform ) if os.path.exists( dependenciesFile ): dependencyResolver.loadDependenciesFromFile( dependenciesFile ) # Now that the Dependency Resolver is all setup and ready to work # We should initialise the product handler productHandler = Buildable.ProductHandler( dependencyResolver ) # Load the platform builds productsFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'product-definitions.yaml' ) productHandler.loadProductInformation( productsFile ) # Initialise the branch resolver branchResolver = Buildable.BranchResolver( dependencyResolver ) # Ask the resolver to load the list branch mapping lmsLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'logical-module-structure' ) branchResolver.loadProjectsToBranchesData( lmsLocation ) # Before we proceed let's do a few sanity checks to make sure we have been called with a valid request # Is this product known? if arguments.product not in productHandler.knownProducts(): print("Fatal exception: The provided product is not known") sys.exit(1) # Is this branch group used with this product? if arguments.branchGroup not in productHandler.branchGroupsFor( arguments.product ): print("Fatal exception: The provided branch group is not enabled for this product") sys.exit(1) # Is this platform enabled for this product? if arguments.platform not in productHandler.platformsFor( arguments.product ): print("Fatal exception: The provided platform is not enabled for this product") sys.exit(1) # Now all the validation is out of the way... print("KDE CI Product Dependency Builder") print("Building For Product: " + arguments.product + "\n") # Determine which projects are built on this platform print("Determining Projects in this Product...") builtProjects = productHandler.projectsFor( arguments.product, arguments.platform ) # Prepare to resolve the dependencies of all the projects projectDependencies = {} initialDependencies = [] cleanedDependencies = [] completeDependencies = [] reverseDependencies = collections.defaultdict(list) # For each project we now resolve it's dependencies print("Resolving Immediate Dependencies for Member Projects...") for project in builtProjects: # Resolve them for this project... ourDependencies = dependencyResolver.forProject( project, arguments.platform ) # Store them for our later sorting projectDependencies[ project.name ] = ourDependencies # Add them the list we've assembled initialDependencies += ourDependencies # Go over and add the necessary items to the reverse dependency map for dependency in ourDependencies: reverseDependencies[ dependency.name ].append( project.name ) # Determine which projects aren't depended on by anyone else # We don't need to build these in order to allow for everything else to be built cleanedDependencies = [ project for project in initialDependencies if project.name in reverseDependencies ] # We will now resolve the full dependency graph of everything outside of the Product # This will ensure we can assemble a fully reliable order to build everything # And won't fail because something outside of the Product depends on something inside the Product # It is acceptable at this point to include something which is in the Product even though that really should be happening # Developers are assumed to accept responsibility for the broken pieces if they break ABI, etc. print("Resolving Dependencies for all Identified Immediate Dependencies") for project in cleanedDependencies: # Resolve the dependencies for this project... ourDependencies = dependencyResolver.forProject( project, arguments.platform ) # Store them for our later sorting projectDependencies[ project.name ] = ourDependencies # Add them to the complete list of items to build completeDependencies += ourDependencies # Also add ourselves completeDependencies.append( project ) # Perform de-duplication on the list of complete dependencies to build print("Eliminating Duplicates from the complete list of Dependencies") completeDependencies = list(set(completeDependencies)) # Now it's time to sort them all into a build order # Make preparations to do so... orderToBuild = [] # Begin the sorting process # To do this we basically compare the projects dependencies against the list of projects we have to build # If all of the dependencies of the project are satisfied we can consider it suitable for building and add it to the list print("Determining Build Order, To Be Built As Follows:") while len(completeDependencies) != 0: # Go over all the products in our list for project in completeDependencies: # Grab the products dependencies ourDependencies = projectDependencies[ project.name ] # Eliminate them against what's in our list of things to build remainingDependencies = list( set(ourDependencies) - set(orderToBuild) ) # Do we have anything left? if len(remainingDependencies) > 0: # Not it's turn unfortunately continue # We have a winner! completeDependencies.remove(project) orderToBuild.append(project) print("== " + project.name) # Now it's time to do the actual process of building # To make things easier, let's find a few things out baseDirectory = os.getcwd() # Make sure we have an installation prefix if not os.path.isdir( arguments.installTo ): os.makedirs( arguments.installTo ) # Let's start! for project in orderToBuild: # Print a notice of where we are at... print("\n\n\n**** Beginning Build For: " + project.name + " ****\n\n\n") # Determine the branch that would be built branchToBuild = branchResolver.branchFor( project, arguments.branchGroup ) # Do we have a valid branch? if branchToBuild is None or branchToBuild == '': print("\n--- Unable to Commence Build: No known branch to build") print("--- Please ensure the kde-build-metadata repository is updated to include branch information for this project!\n\n") continue # Do we need to pass any special parameters to Git? gitCommandParameters = "" # On Windows we do not want Symlinks as many tools just can't handle them and die horribly if sys.platform == 'win32': gitCommandParameters = "-c core.symlinks=false" # All of the below will be run inside a try block try: # Clone the necessary Git repository gitCloneCommand = 'git clone {gitCommandParameters} "{repositoryUrl}" --branch "{branchToBuild}"' commandToRun = gitCloneCommand.format( repositoryUrl='git://anongit.kde.org/' + project.name, branchToBuild=branchToBuild, gitCommandParameters=gitCommandParameters ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=baseDirectory ) # Construct the path to the directory where the repository was cloned to repositoryPath = os.path.join( baseDirectory, project.name ) # Are we on Android? # If so, we may need to build system dependencies of the project we are about to try to build if 'QT_ANDROID' in os.environ and sys.platform == 'linux': # Perform the system dependency build... systemDependencyCommand = '"{baseDirectory}/ci-tooling/helpers/build-android-system-dependency.sh" {productName} {projectName} {currentPlatform}' commandToRun = systemDependencyCommand.format( baseDirectory=baseDirectory, productName=arguments.product, projectName=project.name, currentPlatform=arguments.platform ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath ) # Configure It configureCommand = '"{pythonExecutable}" -u "{baseDirectory}/ci-tooling/helpers/configure-build.py" --product {productName} --project {projectName} --branchGroup {branchGroup} --platform {currentPlatform} --installTo "{installTo}"' commandToRun = configureCommand.format( pythonExecutable=sys.executable, baseDirectory=baseDirectory, productName=arguments.product, projectName=project.name, branchGroup=arguments.branchGroup, currentPlatform=arguments.platform, installTo=arguments.installTo ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath ) # Compile It compileCommand = '"{pythonExecutable}" -u "{baseDirectory}/ci-tooling/helpers/compile-build.py" --product {productName} --project {projectName} --branchGroup {branchGroup} --platform {currentPlatform} --usingInstall "{installTo}"' commandToRun = compileCommand.format( pythonExecutable=sys.executable, baseDirectory=baseDirectory, productName=arguments.product, projectName=project.name, branchGroup=arguments.branchGroup, currentPlatform=arguments.platform, installTo=arguments.installTo ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath ) # Install It installCommand = '"{pythonExecutable}" -u "{baseDirectory}/ci-tooling/helpers/install-build.py" --product {productName} --project {projectName} --branchGroup {branchGroup} --platform {currentPlatform} --installTo "{installTo}" --divertTo "{baseDirectory}/{projectName}/install-divert/"' commandToRun = installCommand.format( pythonExecutable=sys.executable, baseDirectory=baseDirectory, productName=arguments.product, projectName=project.name, branchGroup=arguments.branchGroup, currentPlatform=arguments.platform, installTo=arguments.installTo ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath ) # Finally Capture It captureCommand = '"{pythonExecutable}" -u "{baseDirectory}/ci-tooling/helpers/capture-install.py" --product {productName} --project {projectName} --branchGroup {branchGroup} --environment {ciEnvironment} --platform {currentPlatform} --divertedTo "{baseDirectory}/{projectName}/install-divert/" --installedTo "{installTo}"' commandToRun = captureCommand.format( pythonExecutable=sys.executable, baseDirectory=baseDirectory, productName=arguments.product, projectName=project.name, branchGroup=arguments.branchGroup, currentPlatform=arguments.platform, installTo=arguments.installTo, ciEnvironment=arguments.environment ) subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=repositoryPath ) # Then clean it up - helps to keep disk usage under control during dependency builds (especially useful when many are running simultaneously) - cleanupCommand = 'rm -rf "{baseDirectory}/{projectName}/"' - commandToRun = cleanupCommand.format( - baseDirectory=baseDirectory, projectName=project.name - ) - subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=baseDirectory ) + # We can't do this on Windows as 'rm' isn't a valid command there, but as the impact of the extra space usage on Windows doesn't matter it doesn't cause an issue if we skip this there + if sys.platform != 'win32': + cleanupCommand = 'rm -rf "{baseDirectory}/{projectName}/"' + commandToRun = cleanupCommand.format( + baseDirectory=baseDirectory, projectName=project.name + ) + subprocess.check_call( commandToRun, stdout=sys.stdout, stderr=sys.stderr, shell=True, cwd=baseDirectory ) except Exception: sys.exit(1) # All finished sys.exit(0)