diff --git a/helpers/gather-jobs.py b/helpers/gather-jobs.py index 4489d1f..d994834 100755 --- a/helpers/gather-jobs.py +++ b/helpers/gather-jobs.py @@ -1,113 +1,116 @@ #!/usr/bin/python3 import os import sys import json import argparse from helperslib import BuildSpecs, CommonUtils, Buildable, Packages # Parse the command line arguments we've been given parser = argparse.ArgumentParser(description='Utility to determine which jobs need to be registered in Jenkins.') parser.add_argument('--environment', type=str, required=True) arguments = parser.parse_args() # Initialise the dependnecy resolver dependencyResolver = Buildable.DependencyResolver() # Ask the resolver to load the list of projects... projectsTreeLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'repo-metadata', 'projects' ) dependencyResolver.loadProjectsFromTree( projectsTreeLocation ) +# Load our local ignore file +ignoreFileLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'ignored-projects' ) +dependencyResolver.loadProjectsIgnoreList( ignoreFileLocation ) # Make sure the Platform specific ignore rules are loaded as well ignoreRulesLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'project-ignore-rules.yaml' ) dependencyResolver.loadProjectsIgnoreRules( ignoreRulesLocation ) # Initialise the product handler productHandler = Buildable.ProductHandler( dependencyResolver ) # Load the platform builds productsFile = os.path.join( CommonUtils.scriptsBaseDirectory(), 'local-metadata', 'product-definitions.yaml' ) productHandler.loadProductInformation( productsFile ) # Initialise the branch resolver branchResolver = Buildable.BranchResolver( dependencyResolver ) # Ask the resolver to load the list branch mapping lmsLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'kde-build-metadata', 'logical-module-structure' ) branchResolver.loadProjectsToBranchesData( lmsLocation ) # Our output will be a list of Dictionaries, containing several keys: # 1) The name # 2) The repository url # 3) The branch to be built # 4) The branch group it belongs to # 5) The platform it should be run against # 6) The description for the resulting job jobsGathered = [] # We also output a second set of data, which contains the structure (Products and the Platforms those are built on) # This is used to produce some of the supporting / infrastructure jobs (for building all the dependencies of a Product for instance) # As well as setup the views in Jenkins jobsStructure = { 'products': list( productHandler.knownProducts() ), 'environment': arguments.environment, 'combinations': [] } # With everything now ready, we determine what products we have and start going from there for product in productHandler.knownProducts(): # We need the list of branch groups we want to be looking at here relevantBranchGroups = productHandler.branchGroupsFor( product ) # As well as the platforms we are interested in platformsToBuild = productHandler.platformsFor( product ) # Begin determining which jobs this product needs # As certain projects can be ignored on the Platform level we have to do this on a Platform by Platform basis for each Product for platform in platformsToBuild: # Determine which projects are built on this product/platform combination builtProjects = productHandler.projectsFor( product, platform ) # Now that we know that, let's begin going over each branch group (as we need to store some platform/branch group combo information too) for branchGroup in relevantBranchGroups: # Prepare the job structure entry for this platform and branch group combination combinationEntry = { 'product': product, 'platform': platform, 'branchGroup': branchGroup } # Now store that entry... jobsStructure['combinations'].append( combinationEntry ) # We don't have to do any further checks at this point, so let's go over each project for project in builtProjects: # Determine the branch that would be built branchToBuild = branchResolver.branchFor( project, branchGroup ) # Do we have a valid branch? if branchToBuild is None or branchToBuild == '': continue # We have a winner! jobEntry = { 'name': project.name, 'product': product, 'repositoryUrl': 'git://anongit.kde.org/' + project.name, 'browserUrl': 'https://cgit.kde.org/' + project.name + '.git', 'branch': branchToBuild, 'branchGroup': branchGroup, 'platform': platform, 'description': project.description, 'environment': arguments.environment } # Store it jobsGathered.append( jobEntry ) # Now output the jobs we've gathered in JSON to disk # This will subsequently be read in by a Jenkins DSL script and turned into Jenkins Jobs filePath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'gathered-jobs.json' ) with open(filePath, 'w') as jobsFile: json.dump( jobsGathered, jobsFile ) # Also output the structure data we've gathered in JSON to disk # This will subsequently be read in by a Jenkins DSL script filePath = os.path.join( CommonUtils.scriptsBaseDirectory(), 'gathered-structure.json' ) with open(filePath, 'w') as structureFile: json.dump( jobsStructure, structureFile ) # All done! sys.exit(0)