Changeset View
Changeset View
Standalone View
Standalone View
bin/utils.py
1 | # -*- coding: utf-8 -*- | 1 | # -*- coding: utf-8 -*- | ||
---|---|---|---|---|---|
2 | """@brief utilities | 2 | """@brief utilities | ||
3 | this file contains some helper functions for emerge | 3 | this file contains some helper functions for emerge | ||
4 | """ | 4 | """ | ||
5 | 5 | | |||
6 | # copyright: | 6 | # copyright: | ||
7 | # Holger Schroeder <holger [AT] holgis [DOT] net> | 7 | # Holger Schroeder <holger [AT] holgis [DOT] net> | ||
8 | # Patrick Spendrin <ps_ml [AT] gmx [DOT] de> | 8 | # Patrick Spendrin <ps_ml [AT] gmx [DOT] de> | ||
9 | # Ralf Habacker <ralf.habacker [AT] freenet [DOT] de> | 9 | # Ralf Habacker <ralf.habacker [AT] freenet [DOT] de> | ||
10 | 10 | | |||
11 | import ctypes | 11 | import ctypes | ||
12 | import datetime | 12 | import datetime | ||
13 | import ftplib | 13 | import ftplib | ||
14 | import hashlib | | |||
15 | import http.client | 14 | import http.client | ||
16 | import inspect | 15 | import inspect | ||
17 | import shutil | 16 | import shutil | ||
18 | import tarfile | 17 | import tarfile | ||
19 | import tempfile | 18 | import tempfile | ||
20 | import urllib.error | 19 | import urllib.error | ||
21 | import urllib.parse | 20 | import urllib.parse | ||
22 | import urllib.request | 21 | import urllib.request | ||
23 | import zipfile | 22 | import zipfile | ||
24 | from operator import itemgetter | 23 | from operator import itemgetter | ||
25 | 24 | | |||
25 | import EmergeHash | ||||
26 | import Notifier.NotificationLoader | 26 | import Notifier.NotificationLoader | ||
27 | from EmergeConfig import * | 27 | from EmergeConfig import * | ||
28 | from EmergeDebug import verbose, setVerbose, info, debug, warning, error, die | 28 | from EmergeDebug import verbose, setVerbose, info, debug, warning, error, die | ||
29 | 29 | | |||
30 | if os.name == 'nt': | 30 | if os.name == 'nt': | ||
31 | pass | 31 | pass | ||
32 | else: | 32 | else: | ||
33 | pass | 33 | pass | ||
Show All 34 Lines | 67 | try: | |||
68 | f = open('NUL:') | 68 | f = open('NUL:') | ||
69 | p = subprocess.Popen( appname, stdout=f, stderr=f ) | 69 | p = subprocess.Popen( appname, stdout=f, stderr=f ) | ||
70 | p.wait() | 70 | p.wait() | ||
71 | return True | 71 | return True | ||
72 | except OSError: | 72 | except OSError: | ||
73 | debug("could not find application %s" % appname, 1) | 73 | debug("could not find application %s" % appname, 1) | ||
74 | return False | 74 | return False | ||
75 | 75 | | |||
76 | | ||||
77 | class TemporaryVerbosity(object): | | |||
78 | """Context handler for temporarily different verbosity""" | | |||
79 | def __init__(self, tempLevel): | | |||
80 | self.prevLevel = verbose() | | |||
81 | setVerbose(tempLevel) | | |||
82 | | ||||
83 | def __enter__(self): | | |||
84 | return self | | |||
85 | | ||||
86 | def __exit__(self, exc_type, exc_value, trback): | | |||
87 | setVerbose(self.prevLevel) | | |||
88 | | ||||
89 | | ||||
90 | def getFiles( urls, destdir, suffix='' , filenames = ''): | 76 | def getFiles( urls, destdir, suffix='' , filenames = ''): | ||
91 | """download files from 'url' into 'destdir'""" | 77 | """download files from 'url' into 'destdir'""" | ||
92 | debug("getfiles called. urls: %s, filenames: %s, suffix: %s" % (urls, filenames, suffix), 1) | 78 | debug("getfiles called. urls: %s, filenames: %s, suffix: %s" % (urls, filenames, suffix), 1) | ||
93 | # make sure distfiles dir exists | 79 | # make sure distfiles dir exists | ||
94 | if ( not os.path.exists( destdir ) ): | 80 | if ( not os.path.exists( destdir ) ): | ||
95 | os.makedirs( destdir ) | 81 | os.makedirs( destdir ) | ||
96 | 82 | | |||
97 | if type(urls) == list: | 83 | if type(urls) == list: | ||
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Line(s) | 158 | def getHttpFile( host, path, destdir, filename ): | |||
198 | with open( os.path.join( destdir, filename ), "wb" ) as f: | 184 | with open( os.path.join( destdir, filename ), "wb" ) as f: | ||
199 | f.write( data ) | 185 | f.write( data ) | ||
200 | return True | 186 | return True | ||
201 | 187 | | |||
202 | def isCrEol(filename): | 188 | def isCrEol(filename): | ||
203 | with open(filename, "rb") as f: | 189 | with open(filename, "rb") as f: | ||
204 | return str(f.readline(),'UTF-8').endswith("\r\n") | 190 | return str(f.readline(),'UTF-8').endswith("\r\n") | ||
205 | 191 | | |||
206 | def checkFilesDigests( downloaddir, filenames, digests=None ): | | |||
207 | """check digest of (multiple) files specified by 'filenames' from 'downloaddir'""" | | |||
208 | if digests != None: | | |||
209 | if type(digests) == list: | | |||
210 | digestList = digests | | |||
211 | elif digests.find("\n") != -1: | | |||
212 | digestList = digests.splitLines() | | |||
213 | else: | | |||
214 | digestList = [digests] | | |||
215 | | ||||
216 | i = 0 | | |||
217 | for filename in filenames: | | |||
218 | debug("checking digest of: %s" % filename, 1) | | |||
219 | pathName = os.path.join( downloaddir, filename ) | | |||
220 | if digests == None: | | |||
221 | digestFileName = pathName + '.sha1' | | |||
222 | if not os.path.exists( digestFileName ): | | |||
223 | digestFileName, _ = os.path.splitext( pathName ) | | |||
224 | digestFileName += '.sha1' | | |||
225 | if not os.path.exists( digestFileName ): | | |||
226 | error("digest validation request for file %s, but no digest file present" % | | |||
227 | pathName) | | |||
228 | return False | | |||
229 | currentHash = digestFileSha1( pathName ) | | |||
230 | with open( digestFileName, "r" ) as f: | | |||
231 | line = f.readline() | | |||
232 | digest = re.search('\\b[0-9a-fA-F]{40}\\b', line) | | |||
233 | if not digest: | | |||
234 | error(" digestFile %s for file %s does not contain a valid SHA1 hash" % (digestFileName, | | |||
235 | pathName,)) | | |||
236 | return False | | |||
237 | digest = digest.group(0) | | |||
238 | if len(digest) != len(currentHash) or digest.find(currentHash) == -1: | | |||
239 | error("SHA1 hash for file %s (%s) does not match (%s)" % (pathName, currentHash, digest)) | | |||
240 | return False | | |||
241 | # digest provided in digests parameter | | |||
242 | else: | | |||
243 | currentHash = digestFileSha1( pathName ) | | |||
244 | digest = digestList[i].strip() | | |||
245 | if len(digest) != len(currentHash) or digest.find(currentHash) == -1: | | |||
246 | error("SHA1 hash for file %s (%s) does not match (%s)" % (pathName, currentHash, digest)) | | |||
247 | return False | | |||
248 | i = i + 1 | | |||
249 | return True | | |||
250 | | ||||
251 | | ||||
252 | def createFilesDigests( downloaddir, filenames ): | | |||
253 | """create digests of (multiple) files specified by 'filenames' from 'downloaddir'""" | | |||
254 | digestList = list() | | |||
255 | for filename in filenames: | | |||
256 | pathName = os.path.join( downloaddir, filename ) | | |||
257 | digest = digestFileSha1( pathName ) | | |||
258 | entry = filename, digest | | |||
259 | digestList.append(entry) | | |||
260 | return digestList | | |||
261 | | ||||
262 | def createDigestFile(path): | | |||
263 | """creates a sha1 diget file""" | | |||
264 | digets = digestFileSha1(path) | | |||
265 | with open(path + ".sha1","wt+") as f: | | |||
266 | f.write("%s\n" % digets) | | |||
267 | | ||||
268 | def printFilesDigests( digestFiles, buildTarget=None): | | |||
269 | size = len( digestFiles ) | | |||
270 | i = 0 | | |||
271 | for (fileName, digest) in digestFiles: | | |||
272 | print("%40s %s" % ( fileName, digest ), end=' ') | | |||
273 | if size == 1: | | |||
274 | if buildTarget == None: | | |||
275 | print(" '%s'" % ( digest )) | | |||
276 | else: | | |||
277 | print("self.targetDigests['%s'] = '%s'" % ( buildTarget, digest )) | | |||
278 | else: | | |||
279 | if buildTarget == None: | | |||
280 | if i == 0: | | |||
281 | print(" ['%s'," % ( digest )) | | |||
282 | elif i == size-1: | | |||
283 | print(" '%s']" % ( digest )) | | |||
284 | else: | | |||
285 | print(" '%s'," % ( digest )) | | |||
286 | i = i + 1 | | |||
287 | else: | | |||
288 | if i == 0: | | |||
289 | print("self.targetDigests['%s'] = ['%s'," % ( buildTarget, digest )) | | |||
290 | elif i == size-1: | | |||
291 | print(" '%s']" % ( digest )) | | |||
292 | else: | | |||
293 | print(" '%s'," % ( digest )) | | |||
294 | i = i + 1 | | |||
295 | 192 | | |||
296 | ### unpack functions | 193 | ### unpack functions | ||
297 | 194 | | |||
298 | def unpackFiles( downloaddir, filenames, workdir ): | 195 | def unpackFiles( downloaddir, filenames, workdir ): | ||
299 | """unpack (multiple) files specified by 'filenames' from 'downloaddir' into 'workdir'""" | 196 | """unpack (multiple) files specified by 'filenames' from 'downloaddir' into 'workdir'""" | ||
300 | cleanDirectory( workdir ) | 197 | cleanDirectory( workdir ) | ||
301 | 198 | | |||
302 | for filename in filenames: | 199 | for filename in filenames: | ||
▲ Show 20 Lines • Show All 160 Lines • ▼ Show 20 Line(s) | |||||
463 | def getFileListFromDirectory( imagedir ): | 360 | def getFileListFromDirectory( imagedir ): | ||
464 | """ create a file list containing hashes """ | 361 | """ create a file list containing hashes """ | ||
465 | ret = [] | 362 | ret = [] | ||
466 | 363 | | |||
467 | myimagedir = imagedir | 364 | myimagedir = imagedir | ||
468 | if ( not imagedir.endswith( "\\" ) ): | 365 | if ( not imagedir.endswith( "\\" ) ): | ||
469 | myimagedir = myimagedir + "\\" | 366 | myimagedir = myimagedir + "\\" | ||
470 | 367 | | |||
368 | algorithm = EmergeHash.HashAlgorithm.SHA256 | ||||
471 | for root, _, files in os.walk( imagedir ): | 369 | for root, _, files in os.walk( imagedir ): | ||
472 | for fileName in files: | 370 | for fileName in files: | ||
473 | ret.append( ( os.path.join( root, fileName ).replace( myimagedir, "" ), digestFile( os.path.join( root, fileName ) ) ) ) | 371 | ret.append( ( os.path.join( root, fileName ).replace( myimagedir, "" ), algorithm.stringPrefix() + EmergeHash.digestFile( os.path.join( root, fileName), algorithm) ) ) | ||
474 | return ret | 372 | return ret | ||
475 | 373 | | |||
476 | 374 | | |||
477 | def unmergeFileList(rootdir, fileList, forced=False): | 375 | def unmergeFileList(rootdir, fileList, forced=False): | ||
478 | """ delete files in the fileList if has matches or forced is True """ | 376 | """ delete files in the fileList if has matches or forced is True """ | ||
479 | for filename, filehash in fileList: | 377 | for filename, filehash in fileList: | ||
480 | fullPath = os.path.join(rootdir, os.path.normcase( filename)) | 378 | fullPath = os.path.join(rootdir, os.path.normcase( filename)) | ||
481 | if os.path.isfile(fullPath): | 379 | if os.path.isfile(fullPath): | ||
482 | currentHash = digestFile(fullPath) | 380 | algorithm = EmergeHash.HashAlgorithm.getAlgorithmFromPrefix(filehash) | ||
381 | if not algorithm: | ||||
382 | currentHash = EmergeHash.digestFile(fullPath, EmergeHash.HashAlgorithm.MD5) | ||||
383 | else: | ||||
384 | currentHash = algorithm.stringPrefix() + EmergeHash.digestFile(fullPath, algorithm) | ||||
483 | if currentHash == filehash or filehash == "": | 385 | if currentHash == filehash or filehash == "": | ||
484 | debug("deleting file %s" % fullPath, 2) | 386 | debug("deleting file %s" % fullPath, 2) | ||
485 | try: | 387 | try: | ||
486 | os.remove(fullPath) | 388 | os.remove(fullPath) | ||
487 | except OSError: | 389 | except OSError: | ||
488 | system( "cmd /C \"attrib -R %s\"" % fullPath ) | 390 | system( "cmd /C \"attrib -R %s\"" % fullPath ) | ||
489 | os.remove(fullPath) | 391 | os.remove(fullPath) | ||
490 | else: | 392 | else: | ||
▲ Show 20 Lines • Show All 108 Lines • ▼ Show 20 Line(s) | 500 | if( os.path.isfile( backup ) ): | |||
599 | os.remove( backup ) | 501 | os.remove( backup ) | ||
600 | 502 | | |||
601 | command = "sed -i.orig %s %s" % ( sedcommand, fileName ) | 503 | command = "sed -i.orig %s %s" % ( sedcommand, fileName ) | ||
602 | 504 | | |||
603 | system( command ) | 505 | system( command ) | ||
604 | finally: | 506 | finally: | ||
605 | os.chdir( olddir ) | 507 | os.chdir( olddir ) | ||
606 | 508 | | |||
607 | def digestFile( filepath ): | | |||
608 | """ md5-digests a file """ | | |||
609 | fileHash = hashlib.md5() | | |||
610 | try: | | |||
611 | with open( filepath, "rb" ) as digFile: | | |||
612 | for line in digFile: | | |||
613 | fileHash.update( line ) | | |||
614 | return fileHash.hexdigest() | | |||
615 | except IOError: | | |||
616 | return "" | | |||
617 | | ||||
618 | | ||||
619 | def digestFileSha1( filepath ): | | |||
620 | """ sha1-digests a file """ | | |||
621 | fileHash = hashlib.sha1() | | |||
622 | with open( filepath, "rb" ) as hashFile: | | |||
623 | for line in hashFile: | | |||
624 | fileHash.update( line ) | | |||
625 | return fileHash.hexdigest() | | |||
626 | | ||||
627 | def getVCSType( url ): | 509 | def getVCSType( url ): | ||
628 | """ return the type of the vcs url """ | 510 | """ return the type of the vcs url """ | ||
629 | if not url: | 511 | if not url: | ||
630 | return "" | 512 | return "" | ||
631 | if isGitUrl( url ): | 513 | if isGitUrl( url ): | ||
632 | return "git" | 514 | return "git" | ||
633 | elif url.find("://") == -1: | 515 | elif url.find("://") == -1: | ||
634 | return "svn" | 516 | return "svn" | ||
▲ Show 20 Lines • Show All 563 Lines • Show Last 20 Lines |