Changeset View
Changeset View
Standalone View
Standalone View
helpers/helperslib/Packages.py
| 1 | import copy | ||||
|---|---|---|---|---|---|
| 1 | import os | 2 | import os | ||
| 2 | import re | 3 | import re | ||
| 3 | import stat | 4 | import stat | ||
| 4 | import yaml | 5 | import yaml | ||
| 5 | import shutil | 6 | import shutil | ||
| 6 | import tempfile | 7 | import tempfile | ||
| 7 | import paramiko | 8 | import paramiko | ||
| 8 | import urllib, urllib.request | 9 | import urllib, urllib.request | ||
| ▲ Show 20 Lines • Show All 120 Lines • ▼ Show 20 Line(s) | 78 | def retrievePackage( self, package ): | |||
| 129 | # As we want to return the metadata already parsed (nobody outside this class needs to know it is stored as YAML) we'll load it now | 130 | # As we want to return the metadata already parsed (nobody outside this class needs to know it is stored as YAML) we'll load it now | ||
| 130 | # The archive is returned as a simple path to the file, which can be passed to tarfile.open() as appropriate | 131 | # The archive is returned as a simple path to the file, which can be passed to tarfile.open() as appropriate | ||
| 131 | with open(localMetadataPath, 'r', encoding='utf-8') as localMetadataFile: | 132 | with open(localMetadataPath, 'r', encoding='utf-8') as localMetadataFile: | ||
| 132 | localMetadata = yaml.load( localMetadataFile ) | 133 | localMetadata = yaml.load( localMetadataFile ) | ||
| 133 | 134 | | |||
| 134 | return ( localContentsPath, localMetadata ) | 135 | return ( localContentsPath, localMetadata ) | ||
| 135 | 136 | | |||
| 136 | # Generates the metadata which is stored in the .yaml file that accompanies each package which is stored in the archive | 137 | # Generates the metadata which is stored in the .yaml file that accompanies each package which is stored in the archive | ||
| 137 | def generateMetadataForFile( self, contentsNeedingMetadata, scmRevision ): | 138 | # Extra metadata saved to metadata file, and will be written to yaml file, needs to be a dict like object | ||
| 139 | def generateMetadataForFile( self, contentsNeedingMetadata, scmRevision, extraMetadata=None ): | ||||
| 138 | # First, determine the timestamp the file was last modified | 140 | # First, determine the timestamp the file was last modified | ||
| 139 | packageTimestamp = os.path.getmtime( contentsNeedingMetadata ) | 141 | packageTimestamp = os.path.getmtime( contentsNeedingMetadata ) | ||
| 140 | # Now the checksum | 142 | # Now the checksum | ||
| 141 | packageChecksum = CommonUtils.generateFileChecksum( contentsNeedingMetadata ) | 143 | packageChecksum = CommonUtils.generateFileChecksum( contentsNeedingMetadata ) | ||
| 142 | # Build the metadata which we'll be writing out | 144 | | ||
| 143 | metadataForPackage = { | 145 | metadataForPackage = {} | ||
| 146 | | ||||
| 147 | # If we have extraMetadata for this Package pre-seed the metadata dictionary | ||||
bcooksley: pressed -> pre-seed? | |||||
| 148 | if extraMetadata: | ||||
| 149 | metadataForPackage = copy.copy(self.extraMetadata) | ||||
| 150 | | ||||
| 151 | # Update/adds the nessary keys, that we want to exist. | ||||
| 152 | metadataForPackage.update({ | ||||
| 144 | 'timestamp': packageTimestamp, | 153 | 'timestamp': packageTimestamp, | ||
| 145 | 'checksum': packageChecksum, | 154 | 'checksum': packageChecksum, | ||
| 146 | 'scmRevision': scmRevision | 155 | 'scmRevision': scmRevision | ||
| 147 | } | 156 | }) | ||
| 148 | 157 | | |||
| 149 | # Write the YAML out to a temporary file | 158 | # Write the YAML out to a temporary file | ||
| 150 | latestMetadata = tempfile.NamedTemporaryFile(delete=False, mode='w', dir=self.temporaryFileLocation()) | 159 | latestMetadata = tempfile.NamedTemporaryFile(delete=False, mode='w', dir=self.temporaryFileLocation()) | ||
| 151 | yaml.dump( metadataForPackage, latestMetadata, default_flow_style=False, allow_unicode=True ) | 160 | yaml.dump( metadataForPackage, latestMetadata, default_flow_style=False, allow_unicode=True ) | ||
| 152 | latestMetadata.close() | 161 | latestMetadata.close() | ||
| 153 | 162 | | |||
| 154 | # Return the name to that temporary file | 163 | # Return the name to that temporary file | ||
| 155 | return latestMetadata.name | 164 | return latestMetadata.name | ||
| 156 | 165 | | |||
| 157 | # Stores a package in the archive, either by creation of or updating of an existing package | 166 | # Stores a package in the archive, either by creation of or updating of an existing package | ||
| 158 | # As part of this process metadata will be generated for the package we are about to add to the archive to assist in caching later on | 167 | # As part of this process metadata will be generated for the package we are about to add to the archive to assist in caching later on | ||
| 159 | # The package and it's metadata will then be uploaded to the remote archive and published, then transferred to our local cache | 168 | # The package and it's metadata will then be uploaded to the remote archive and published, then transferred to our local cache | ||
| 160 | def storePackage( self, package, archiveFileToInclude, scmRevision = '' ): | 169 | # Extra metadata saved to metadata file, and will be written to yaml file, needs to be a dict like object | ||
| 170 | def storePackage( self, package, archiveFileToInclude, scmRevision = '', extraMetadata=None ): | ||||
| 161 | # Determine the names the metadata and archive files would have respectively | 171 | # Determine the names the metadata and archive files would have respectively | ||
| 162 | metadataFilename = package + ".yaml" | 172 | metadataFilename = package + ".yaml" | ||
| 163 | contentsFilename = package + ".tar" | 173 | contentsFilename = package + ".tar" | ||
| 164 | 174 | | |||
| 165 | # Generate metadata for the package we are about to store | 175 | # Generate metadata for the package we are about to store | ||
| 166 | archiveMetadata = self.generateMetadataForFile( archiveFileToInclude, scmRevision ) | 176 | archiveMetadata = self.generateMetadataForFile( archiveFileToInclude, scmRevision, extraMetadata ) | ||
| 167 | 177 | | |||
| 168 | # Connect to the upload host | 178 | # Connect to the upload host | ||
| 169 | privateKeyFile = os.path.join( os.path.expanduser('~'), 'Keys', self.name + '.key') | 179 | privateKeyFile = os.path.join( os.path.expanduser('~'), 'Keys', self.name + '.key') | ||
| 170 | uploadConnection = CommonUtils.establishSSHConnection( self.config['client']['uploadHostname'], self.config['client']['uploadUsername'], privateKeyFile ) | 180 | uploadConnection = CommonUtils.establishSSHConnection( self.config['client']['uploadHostname'], self.config['client']['uploadUsername'], privateKeyFile ) | ||
| 171 | 181 | | |||
| 172 | # Initiate SFTP uploads | 182 | # Initiate SFTP uploads | ||
| 173 | sftp = uploadConnection.open_sftp() | 183 | sftp = uploadConnection.open_sftp() | ||
| 174 | # Upload our archive and it's metadata file | 184 | # Upload our archive and it's metadata file | ||
| ▲ Show 20 Lines • Show All 100 Lines • Show Last 20 Lines | |||||
pressed -> pre-seed?