CVS: src/si/extras entitycheck.py,1.17,1.18
Mats Wichmann <[email protected]>
| Newsgroups | gmane.linux.lsb.implementation |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/lsb/src/si/extras
In directory sc8-pr-cvs1:/tmp/cvs-serv11548
Modified Files:
entitycheck.py
Log Message:
Lots more tweaking.
Index: entitycheck.py
===================================================================
RCS file: /cvsroot/lsb/src/si/extras/entitycheck.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -C2 -r1.17 -r1.18
*** entitycheck.py 8 Jan 2003 00:02:56 -0000 1.17
--- entitycheck.py 8 Jan 2003 17:12:40 -0000 1.18
***************
*** 17,21 ****
-s FILE, --sumfile=FILE -- use md5sum file FILE [%s]
-f, --fetch -- fetch missing pkgs
! --dryrun -- test what pkgs would be retrived
-u FILE, --updatefile=FILE -- use FILE for pkg locations [%s]
-z URL, --fallback=URL -- use URL for fallback for pkgs [%s]
--- 17,21 ----
-s FILE, --sumfile=FILE -- use md5sum file FILE [%s]
-f, --fetch -- fetch missing pkgs
! --dryrun -- test what pkgs would be retrieved
-u FILE, --updatefile=FILE -- use FILE for pkg locations [%s]
-z URL, --fallback=URL -- use URL for fallback for pkgs [%s]
***************
*** 46,91 ****
sys.exit(code)
- shortopts = 'qe:p:d:gcs:fu:z:h'
- longopts = ['quiet', 'entityfile=', 'packagepath=', 'patchpath=',
- 'gensum', 'checksum', 'sumfile=', 'fetch', 'updatefile=',
- 'dryrun', 'fallback=', 'help']
- try:
- opts, args = getopt.getopt(sys.argv[1:], shortopts, longopts)
- except getopt.error, msg:
- usage(2, msg)
-
- if opts:
- for (opt, arg) in opts:
- if opt in ('--help', '-h'):
- usage(0)
- if opt in ('--entityfile', '-e'):
- entity_file = arg
- if opt in ('--packagepath', '-p'):
- package_path = arg
- if opt in ('--patchpath', '-d'):
- patch_path = arg
- if opt in ('--gensum', '-g'):
- generate_sums = 'yes'
- if opt in ('--checksum', '-c'):
- check_sums = 'yes'
- if opt in ('--sumfile', '-s'):
- md5sum_file = arg
- if opt in ('--fetch', '-f'):
- fetch_files = 'yes'
- if opt in ('--updatefile', '-u'):
- update_file = arg
- if opt == '--dryrun':
- dry_run = 'yes'
- fetch_files = 'yes'
- if opt in ('--fallback', '-z'):
- fallback_url = arg
- if opt in ('--quiet', '-q'):
- noisy = None
-
# these two functions print messages during file retrieval
initial_text = "" # needs to be global so feedback() can access it
def running_output(front, message):
! """Print running output of front, message using \b and \r tricks."""
output = "%s %s" % (front, message)
padding = " " * (80 - len(output))
--- 46,54 ----
sys.exit(code)
# these two functions print messages during file retrieval
initial_text = "" # needs to be global so feedback() can access it
def running_output(front, message):
! """Print running output of (front, message) using \b and \r tricks."""
output = "%s %s" % (front, message)
padding = " " * (80 - len(output))
***************
*** 101,104 ****
--- 64,70 ----
class entity:
+ """entity class, an instance is created for each entity read
+ from the nALFS entity file
+ """
def __init__(self, name, file):
self.name = name
***************
*** 106,163 ****
def domd5(self):
! """Generate md5sum for this entity's filename, and save"""
! f = open(self.fullpath, "rb")
! sum = md5.new()
! while 1:
! block = f.read(BLOCKSIZE)
! if not block:
! break
! sum.update(block)
! f.close()
! s = sum.digest()
! self.md5sum = "%02x"*len(s) % tuple(map(ord, s))
def fetch(self, locations, fallback, destination):
! """retrieve a missing or bad file.
! Calls urllib.urlretrieve() up to three times to retrieve to
! destination from 1. main location, 2. alternate (if any),
! 3. fallback location
! """
! global initial_text
! message = "not found"
! for loc in locations:
! initial_text = "%s ->" % self.name
! if self.name != loc.name: continue
! pkgpath = "%s/%s" % (destination, self.file)
! print initial_text,
! sys.stdout.flush() # force text to be displayed
!
! # try up to three times to fetch the file
! if dry_run:
! print "fetch %s from (%s, %s, %s) to %s" % (self.file,
! loc.path, loc.alternate, fallback, destination)
! message = "skipped"
! break
! for url in loc.path, loc.alternate, fallback:
! if not url: continue # skip alternate if not defined
! to_get = "%s/%s" % (url, self.file)
!
! try:
! filename, mime = urllib.urlretrieve(to_get, pkgpath, feedback)
! if mime and mime.gettype() == 'text/html':
! raise IOError
! message = "completed"
! break
! except IOError:
! if os.path.exists(pkgpath): os.remove(pkgpath)
! else: # if we didn't "break" out of loop, fetch failed
! message = "retrieval failed"
! break
!
! running_output(initial_text, message)
! print
! return message
! def parse_entities(file):
"""Parse the entities file. Returns a tuple containing a list
of package entities and a list of patch entities.
--- 72,131 ----
def domd5(self):
! """Generate md5sum for this entity's filename, and save"""
! f = open(self.fullpath, "rb")
! sum = md5.new()
! while 1:
! block = f.read(BLOCKSIZE)
! if not block:
! break
! sum.update(block)
! f.close()
! s = sum.digest()
! self.md5sum = "%02x"*len(s) % tuple(map(ord, s))
def fetch(self, locations, fallback, destination):
! """retrieve the file for an entity.
! Calls urllib.urlretrieve() up to three times to retrieve to
! destination from main or alternate location as found
! in the 'locations' entity list, or if neither works from
! the 'fallback'. Returns a string indicating status.
! """
! global initial_text
! message = "not found"
! for loc in locations:
! initial_text = "%s ->" % self.name
! if self.name != loc.name: continue
! pkgpath = "%s/%s" % (destination, self.file)
! print initial_text,
! sys.stdout.flush() # force text to be displayed
!
! if dry_run: # just print a message and bail
! print "fetch %s from (%s, %s, %s) to %s" % (self.file,
! loc.path, loc.alternate, fallback, destination)
! message = "skipped"
! break
!
! # try up to three times to fetch the file
! for url in loc.path, loc.alternate, fallback:
! if not url: continue # skip alternate if not defined
! to_get = "%s/%s" % (url, self.file)
!
! try:
! filename, mime = urllib.urlretrieve(to_get, pkgpath, feedback)
! if mime and mime.gettype() == 'text/html':
! raise IOError
! message = "completed"
! break
! except IOError:
! if os.path.exists(pkgpath): os.remove(pkgpath)
! else: # if we didn't "break" out of loop, fetch failed
! message = "retrieval failed"
! break
!
! running_output(initial_text, message)
! print
! return message
! def parse_entities():
"""Parse the entities file. Returns a tuple containing a list
of package entities and a list of patch entities.
***************
*** 189,196 ****
class location:
def __init__(self, name, path, alternate=None):
! self.name = name
! self.path = path
! self.alternate = alternate
def parse_locations(file):
--- 157,165 ----
class location:
+ """location instances are created for each line in the locations file"""
def __init__(self, name, path, alternate=None):
! self.name = name
! self.path = path
! self.alternate = alternate
def parse_locations(file):
***************
*** 211,223 ****
bits = string.split(line)
if len(bits) < 3 or bits[1] == "none" or bits[1] == "None":
! # skip short lines, or those with no retrieve location
continue
! if bits[2] == "none" or bits[2] == "None": bits[2] = None
! locations.append(location(bits[0], bits[1], bits[2]))
return locations
def check_missing(path, collection):
! """Scan a collection of entities, returning a tuple consisting of a
! list of found entities and a list of missing entities.
Generate checksums for found entities, if requested.
"""
--- 180,193 ----
bits = string.split(line)
if len(bits) < 3 or bits[1] == "none" or bits[1] == "None":
! # skip short lines, or those with no retrieve location
continue
! if bits[2] == "none" or bits[2] == "None": bits[2] = None
! locations.append(location(bits[0], bits[1], bits[2]))
! package_file.close()
return locations
def check_missing(path, collection):
! """Scan a collection of entities, returning a tuple
! (list of found entities, list of missing entities).
Generate checksums for found entities, if requested.
"""
***************
*** 230,241 ****
missing.append(item)
else:
if check_sums or generate_sums:
item.domd5()
- found.append(item)
return (found, missing)
def check_extra(path, collection):
"""Check for files in a path that are not described by entities.
! Returns a list of dummy entities, with only the file attribute defined
"""
notfound = []
--- 200,212 ----
missing.append(item)
else:
+ found.append(item)
if check_sums or generate_sums:
item.domd5()
return (found, missing)
def check_extra(path, collection):
"""Check for files in a path that are not described by entities.
! Creates an entity instance for each and returns a list (this
! is to be able to use a common print routine, only the names matter)
"""
notfound = []
***************
*** 250,256 ****
def check_checksums(collection, checksums):
"""Check checksums on entities in collection against 'checksums' dictionary.
! Returns a list of entities with bad checksums.
"""
badsums = []
for entity in collection:
if checksums.has_key(entity.file):
--- 221,228 ----
def check_checksums(collection, checksums):
"""Check checksums on entities in collection against 'checksums' dictionary.
! Returns a tuple (entities with bad checksums, missing checksums)
"""
badsums = []
+ nosums = []
for entity in collection:
if checksums.has_key(entity.file):
***************
*** 258,304 ****
badsums.append(entity)
else:
! print "Warning: Checksum file may be out of date,",
! print "no entry for", entity.file
! return badsums
def dump_coll(collection, msg):
if collection:
! print msg, len(collection)
! for item in collection:
! print "\t", item.file
! def report(fnd_pkg, fnd_pat, miss_pkg, miss_pat, extras, bad_pkg, bad_pat):
"""Generate package/patch/checksum report.
Global "noisy" controls whether there's any output
Return non-zero on fatal error (missing files or bad sums)
"""
if noisy:
print "Package entities found:", len(fnd_pkg)
print "Patch entities found:", len(fnd_pat)
! dump_coll(miss_pkg, "Packages not found:")
! dump_coll(miss_pat, "Patches not found:")
! dump_coll(bad_pkg, "Bad packages:")
! dump_coll(bad_pat, "Bad patches:")
! dump_coll(extras, "Files not in use:")
! if miss_pkg or miss_pat or bad_pkg or bad_pat:
! return 1
! else:
! return 0
! def fetch_report(retrieved, missed, missing):
! """Generate file fetch report. Split from regular report
! because we want report to show, then the fetch attempt,
! then the fetch report.
Global "noisy" controls whether there's any output
Return non-zero on fatal error (missing files or bad sums)
"""
if noisy:
print "Updated %s packages" % retrieved
! dump_coll(missed, "Packages which failed to retrieve:")
! dump_coll(missing, "Entities missing:")
! if missed or missing:
! return 1
else:
! return 0
def readmd5(md5sum_file):
--- 230,284 ----
badsums.append(entity)
else:
! badsums.append(entity)
! return (badsums, nosums)
def dump_coll(collection, msg):
if collection:
! print msg, len(collection)
! for item in collection:
! print "\t", item.file
! # For debugging: print both name and file
! #print "\t", item.name, "->", item.file
! def report(fnd_pkg, fnd_pat, miss_pkg, miss_pat, extras, *sums):
"""Generate package/patch/checksum report.
Global "noisy" controls whether there's any output
Return non-zero on fatal error (missing files or bad sums)
"""
+ rv = 0
if noisy:
+ print 'Checked entities from', entity_file
+ print 'Checked packages from', package_path
+ print 'Checked patches from', patch_path
print "Package entities found:", len(fnd_pkg)
+ dump_coll(miss_pkg, "Missing packages:")
print "Patch entities found:", len(fnd_pat)
! dump_coll(miss_pat, "Missing patches:")
! if miss_pkg or miss_pat: rv = 1
! if check_sums:
! print "Checked checksums against", md5sum_file
! dump_coll(sums[0], "Packages with bad checksums:")
! dump_coll(sums[1], "Patches with bad checksums:")
! dump_coll(sums[2], "Packages without checksums:")
! dump_coll(sums[3], "Patches without checksums:")
! if sums[0] or sums[1]: rv = 1
! dump_coll(extras, "Files not in use:")
! return rv
! def fetch_report(retrieved, failed, missing):
! """Generate file fetch report.
Global "noisy" controls whether there's any output
Return non-zero on fatal error (missing files or bad sums)
"""
+ # "fetch" report split from regular report because we want report
+ # to show, then the fetch attempt, then the fetch report.
if noisy:
print "Updated %s packages" % retrieved
! dump_coll(failed, "Packages which failed to retrieve:")
! dump_coll(missing, "Entities missing:")
! if failed or missing:
! return 1
else:
! return 0
def readmd5(md5sum_file):
***************
*** 326,357 ****
f.write("%s %s\n" % (entity.md5sum, entity.file))
! def retrieve_packages(missing_packages, file, fallback, destination):
"""Retrieve packages identified as missing.
"""
! locations = parse_locations(file)
retrieved = 0
! missed = []
! missing = []
global initial_text
for pkg in missing_packages:
! rv = pkg.fetch(locations, fallback, destination)
! if rv == 'completed':
! retrieved = retrieved + 1
! if rv == 'retrieval failed':
! missed.append(pkg)
! if rv == 'not found':
! # should not happen, but we track so we can catch where
! # an entity is defined but not listed in package_locations
! missing.append(pkg)
! return (retrieved, missed, missing)
!
! # Main program:
! if noisy:
! print 'Checking entities from', entity_file
! print 'Checking packages from', package_path
! print 'Checking patches from', patch_path
! if check_sums:
! print "Checking checksums against", md5sum_file
if not os.path.isdir(package_path):
usage(1, "Path to packages <%s> is invalid" % package_path)
--- 306,357 ----
f.write("%s %s\n" % (entity.md5sum, entity.file))
! def retrieve_packages(missing_packages):
"""Retrieve packages identified as missing.
"""
! locations = parse_locations(update_file)
retrieved = 0
! fails, missing = [], []
global initial_text
for pkg in missing_packages:
! rv = pkg.fetch(locations, fallback_url, package_path)
! if rv == 'completed':
! retrieved = retrieved + 1
! if rv == 'retrieval failed':
! fails.append(pkg)
! if rv == 'not found':
! # should not happen, but we track so we can catch where
! # an entity is defined but not listed in package_locations
! missing.append(pkg)
! return (retrieved, fails, missing)
!
! ## Main
! # 1. Process command-line arguments
! shortopts = 'qe:p:d:gcs:fu:z:h'
! longopts = ['quiet', 'entityfile=', 'packagepath=', 'patchpath=',
! 'gensum', 'checksum', 'sumfile=', 'fetch', 'updatefile=',
! 'dryrun', 'fallback=', 'help']
! try:
! opts, args = getopt.getopt(sys.argv[1:], shortopts, longopts)
! except getopt.error, msg:
! usage(2, msg)
+ if opts:
+ for (opt, arg) in opts:
+ if opt in ('--help', '-h'): usage(0)
+ if opt in ('--entityfile', '-e'): entity_file = arg
+ if opt in ('--packagepath', '-p'): package_path = arg
+ if opt in ('--patchpath', '-d'): patch_path = arg
+ if opt in ('--gensum', '-g'): generate_sums = 'yes'
+ if opt in ('--checksum', '-c'): check_sums = 'yes'
+ if opt in ('--sumfile', '-s'): md5sum_file = arg
+ if opt in ('--fetch', '-f'): fetch_files = 'yes'
+ if opt in ('--updatefile', '-u'): update_file = arg
+ if opt == '--dryrun':
+ dry_run = 'yes'
+ fetch_files = 'yes'
+ if opt in ('--fallback', '-z'): fallback_url = arg
+ if opt in ('--quiet', '-q'): noisy = None
+
+ # 2. Check directories are okay up front
if not os.path.isdir(package_path):
usage(1, "Path to packages <%s> is invalid" % package_path)
***************
*** 359,364 ****
usage(1, "Path to packages <%s> is invalid" % patch_path)
! packages, patches = parse_entities(entity_file)
!
found_packages, missing_packages = check_missing(package_path, packages)
found_patches, missing_patches = check_missing(patch_path, patches)
--- 359,363 ----
usage(1, "Path to packages <%s> is invalid" % patch_path)
! packages, patches = parse_entities()
found_packages, missing_packages = check_missing(package_path, packages)
found_patches, missing_patches = check_missing(patch_path, patches)
***************
*** 366,407 ****
if package_path == patch_path:
extras = check_extra(package_path, found_packages + found_patches)
! else:
extras = check_extra(package_path, found_packages)
extras = extras + check_extra(patch_path, found_patches)
- bad_packages = None
- bad_patches = None
if check_sums:
checksums = readmd5(md5sum_file)
! bad_packages = check_checksums(found_packages, checksums)
! bad_patches = check_checksums(found_patches, checksums)
!
! # tell us what happened
! exitcode = report(found_packages, found_patches, missing_packages,
! missing_patches, extras, bad_packages, bad_patches)
# go fetch files if needed
if fetch_files:
retrieved = 0
! missed = missing = []
! if bad_packages:
! retrieved, missed, missing = retrieve_packages(bad_packages,
! update_file, fallback_url, package_path)
if missing_packages:
! r2, m2, x2 = retrieve_packages(missing_packages,
! update_file, fallback_url, package_path)
! retrieved = retrieved + r2
! missed = missed + m2
! missing = missing + x2
# tell us what happened on the fetch
! exitcode = exitcode + fetch_report(retrieved, missed, missing)
# quit non-zero if any entities were missing
if exitcode:
! if generate_sums and noisy:
print "Not writing checksum file, errors found"
sys.exit(exitcode)
if generate_sums:
writemd5(found_packages + found_patches)
--- 365,408 ----
if package_path == patch_path:
extras = check_extra(package_path, found_packages + found_patches)
! else: # packages and patches in separate directories
extras = check_extra(package_path, found_packages)
extras = extras + check_extra(patch_path, found_patches)
if check_sums:
checksums = readmd5(md5sum_file)
! bad_packages, no_packages = check_checksums(found_packages, checksums)
! bad_patches, no_patches = check_checksums(found_patches, checksums)
! # tell us what happened
! exitcode = report(found_packages, found_patches,
! missing_packages, missing_patches, extras,
! bad_packages, bad_patches, no_packages, no_patches)
! else:
! # tell us what happened, without the checksum stuff
! exitcode = report(found_packages, found_patches,
! missing_packages, missing_patches, extras)
# go fetch files if needed
if fetch_files:
retrieved = 0
! fails, missing = [], []
if missing_packages:
! retrieved, fails, missing = retrieve_packages(missing_packages)
! # if we checked checksums, there may also be pkgs with bad cksums:
! if check_sums and bad_packages:
! r, f, m = retrieve_packages(bad_packages)
! retrieved = retrieved + r
! fails = fails + f
! missing = missing + m
# tell us what happened on the fetch
! exitcode = exitcode + fetch_report(retrieved, fails, missing)
# quit non-zero if any entities were missing
if exitcode:
! if generate_sums:
print "Not writing checksum file, errors found"
sys.exit(exitcode)
+ # if no errors, it's okay to go ahead and generate the new checksum file
if generate_sums:
writemd5(found_packages + found_patches)