Changeset View
Changeset View
Standalone View
Standalone View
helpers/helperslib/Packages.py
| Show All 9 Lines | |||||
| 10 | from helperslib import CommonUtils | 10 | from helperslib import CommonUtils | ||
| 11 | 11 | | |||
| 12 | def nameForProject( product, project, branchGroup ): | 12 | def nameForProject( product, project, branchGroup ): | ||
| 13 | return "{0}-{1}-{2}".format( product, project, branchGroup ) | 13 | return "{0}-{1}-{2}".format( product, project, branchGroup ) | ||
| 14 | 14 | | |||
| 15 | class Archive(object): | 15 | class Archive(object): | ||
| 16 | # Sets up an archive for use, to allow for retrieving and uploading new resources to the archive | 16 | # Sets up an archive for use, to allow for retrieving and uploading new resources to the archive | ||
| 17 | # A local cache of the archive will be managed to improve performance | 17 | # A local cache of the archive will be managed to improve performance | ||
| 18 | def __init__( self, name, platform, usingCache = True ): | 18 | def __init__( self, name, platform, usingCache = True, suffix = ".tar" ): | ||
| 19 | # Save the name of this archive, the platform we're on and whether a local cache should be used for later use | 19 | # Save the name of this archive, the platform we're on and whether a local cache should be used for later use | ||
| 20 | self.name = name | 20 | self.name = name | ||
| 21 | self.platform = platform | 21 | self.platform = platform | ||
| 22 | self.usingCache = usingCache | 22 | self.usingCache = usingCache | ||
| 23 | self.suffix = suffix | ||||
bcooksley: Could you please align the = with the other three ones there?
Also, in terms of variable name… | |||||
IMO don't like the align at =, as this make the diff bigger than necessary. knauss: IMO don't like the align at =, as this make the diff bigger than necessary. | |||||
| 23 | # Set an empty manifest up for safety later on | 24 | # Set an empty manifest up for safety later on | ||
| 24 | self.serverManifest = {} | 25 | self.serverManifest = {} | ||
| 25 | 26 | | |||
| 26 | # Load our configuration | 27 | # Load our configuration | ||
| 27 | configFileLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'archive-configs', name + '.yaml' ) | 28 | configFileLocation = os.path.join( CommonUtils.scriptsBaseDirectory(), 'archive-configs', name + '.yaml' ) | ||
| 28 | with open(configFileLocation, 'r') as configFile: | 29 | with open(configFileLocation, 'r') as configFile: | ||
| 29 | # Parse the YAML file | 30 | # Parse the YAML file | ||
| 30 | self.config = yaml.load( configFile ) | 31 | self.config = yaml.load( configFile ) | ||
| ▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Line(s) | |||||
| 73 | 74 | | |||
| 74 | # Retrieve the package for the given project and branch group combination from the archive | 75 | # Retrieve the package for the given project and branch group combination from the archive | ||
| 75 | # If a cached copy is available and is the most recent version that will be used instead of fetching the package again | 76 | # If a cached copy is available and is the most recent version that will be used instead of fetching the package again | ||
| 76 | # Where the remote archive has a newer version then the package will be retrieved from the remote archive | 77 | # Where the remote archive has a newer version then the package will be retrieved from the remote archive | ||
| 77 | # All lookups will be restricted to our current platform, as specified when creating this archive. | 78 | # All lookups will be restricted to our current platform, as specified when creating this archive. | ||
| 78 | def retrievePackage( self, package ): | 79 | def retrievePackage( self, package ): | ||
| 79 | # Determine the names the metadata and archive files would have respectively | 80 | # Determine the names the metadata and archive files would have respectively | ||
| 80 | metadataFilename = package + ".yaml" | 81 | metadataFilename = package + ".yaml" | ||
| 81 | contentsFilename = package + ".tar" | 82 | contentsFilename = package + self.suffix | ||
| 82 | 83 | | |||
| 83 | # Begin determining if we need to download or not | 84 | # Begin determining if we need to download or not | ||
| 84 | # We start from the assumption we will need to download an archive | 85 | # We start from the assumption we will need to download an archive | ||
| 85 | needToDownload = True | 86 | needToDownload = True | ||
| 86 | 87 | | |||
| 87 | # Do we have a local copy of this? | 88 | # Do we have a local copy of this? | ||
| 88 | localMetadataPath = os.path.join(self.cacheLocation(), metadataFilename) | 89 | localMetadataPath = os.path.join(self.cacheLocation(), metadataFilename) | ||
| 89 | if os.path.exists( localMetadataPath ): | 90 | if os.path.exists( localMetadataPath ): | ||
| ▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Line(s) | |||||
| 166 | 167 | | |||
| 167 | # Stores a package in the archive, either by creation of or updating of an existing package | 168 | # Stores a package in the archive, either by creation of or updating of an existing package | ||
| 168 | # As part of this process metadata will be generated for the package we are about to add to the archive to assist in caching later on | 169 | # As part of this process metadata will be generated for the package we are about to add to the archive to assist in caching later on | ||
| 169 | # The package and it's metadata will then be uploaded to the remote archive and published, then transferred to our local cache | 170 | # The package and it's metadata will then be uploaded to the remote archive and published, then transferred to our local cache | ||
| 170 | # Extra metadata saved to metadata file, and will be written to yaml file, needs to be a dict like object | 171 | # Extra metadata saved to metadata file, and will be written to yaml file, needs to be a dict like object | ||
| 171 | def storePackage( self, package, archiveFileToInclude, scmRevision = '', extraMetadata=None ): | 172 | def storePackage( self, package, archiveFileToInclude, scmRevision = '', extraMetadata=None ): | ||
| 172 | # Determine the names the metadata and archive files would have respectively | 173 | # Determine the names the metadata and archive files would have respectively | ||
| 173 | metadataFilename = package + ".yaml" | 174 | metadataFilename = package + ".yaml" | ||
| 174 | contentsFilename = package + ".tar" | 175 | contentsFilename = package + self.suffix | ||
| 175 | 176 | | |||
| 176 | # Generate metadata for the package we are about to store | 177 | # Generate metadata for the package we are about to store | ||
| 177 | archiveMetadata = self.generateMetadataForFile( archiveFileToInclude, scmRevision, extraMetadata ) | 178 | archiveMetadata = self.generateMetadataForFile( archiveFileToInclude, scmRevision, extraMetadata ) | ||
| 178 | 179 | | |||
| 179 | # Connect to the upload host | 180 | # Connect to the upload host | ||
| 180 | privateKeyFile = os.path.join( os.path.expanduser('~'), 'Keys', self.name + '.key') | 181 | privateKeyFile = os.path.join( os.path.expanduser('~'), 'Keys', self.name + '.key') | ||
| 181 | uploadConnection = CommonUtils.establishSSHConnection( self.config['client']['uploadHostname'], self.config['client']['uploadUsername'], privateKeyFile ) | 182 | uploadConnection = CommonUtils.establishSSHConnection( self.config['client']['uploadHostname'], self.config['client']['uploadUsername'], privateKeyFile ) | ||
| 182 | 183 | | |||
| Show All 31 Lines | |||||
| 214 | with open(localMetadataPath, 'r', encoding='utf-8') as localMetadataFile: | 215 | with open(localMetadataPath, 'r', encoding='utf-8') as localMetadataFile: | ||
| 215 | self.serverManifest[ package ] = yaml.load( localMetadataFile ) | 216 | self.serverManifest[ package ] = yaml.load( localMetadataFile ) | ||
| 216 | 217 | | |||
| 217 | # Performs the package publishing process | 218 | # Performs the package publishing process | ||
| 218 | # This function should only be called on the archive server and will not function correctly on clients. | 219 | # This function should only be called on the archive server and will not function correctly on clients. | ||
| 219 | def publishPackage( self, package ): | 220 | def publishPackage( self, package ): | ||
| 220 | # Determine the names the metadata and archive files would have respectively | 221 | # Determine the names the metadata and archive files would have respectively | ||
| 221 | metadataFilename = package + ".yaml" | 222 | metadataFilename = package + ".yaml" | ||
| 222 | contentsFilename = package + ".tar" | 223 | contentsFilename = package + self.suffix | ||
| 223 | 224 | | |||
| 224 | # Move the contents file first | 225 | # Move the contents file first | ||
| 225 | # Assuming we're on the same file system this should be an atomic operation and thus instant | 226 | # Assuming we're on the same file system this should be an atomic operation and thus instant | ||
| 226 | # We move the metadata second in case uploadDirectory and archiveDirectory are on different file systems | 227 | # We move the metadata second in case uploadDirectory and archiveDirectory are on different file systems | ||
| 227 | # As the contents file could be several hundred megabytes, while the metadata file should be a matter of a few kilobytes and thus copy across instantly | 228 | # As the contents file could be several hundred megabytes, while the metadata file should be a matter of a few kilobytes and thus copy across instantly | ||
| 228 | # Also, as the metadata file governs when files should be expired, it is better to over-expire than risk an outdated cached copy being used | 229 | # Also, as the metadata file governs when files should be expired, it is better to over-expire than risk an outdated cached copy being used | ||
| 229 | stagedContentsPath = os.path.join( self.config['client']['uploadDirectory'], self.platform, contentsFilename ) | 230 | stagedContentsPath = os.path.join( self.config['client']['uploadDirectory'], self.platform, contentsFilename ) | ||
| 230 | finalContentsPath = os.path.join( self.config['server']['archiveDirectory'], self.platform, contentsFilename ) | 231 | finalContentsPath = os.path.join( self.config['server']['archiveDirectory'], self.platform, contentsFilename ) | ||
| ▲ Show 20 Lines • Show All 55 Lines • Show Last 20 Lines | |||||
Could you please align the = with the other three ones there?
Also, in terms of variable name, could we have something that makes it clear what this suffix is for?
(something like self.resourceSuffix perhaps?)