userland-fetch revision 3979
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#!/usr/bin/python2.7
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# CDDL HEADER START
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# The contents of this file are subject to the terms of the
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# Common Development and Distribution License (the "License").
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# You may not use this file except in compliance with the License.
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# or http://www.opensolaris.org/os/licensing.
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# See the License for the specific language governing permissions
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# and limitations under the License.
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# When distributing Covered Code, include this CDDL HEADER in each
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# If applicable, add the following below this CDDL HEADER, with the
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# fields enclosed by brackets "[]" replaced with your own identifying
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# information: Portions Copyright [yyyy] [name of copyright owner]
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#
0b6a21892fd667342bd302336bc327e93e365bdavboxsync# CDDL HEADER END
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync#
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# fetch.py - a file download utility
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync#
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# A simple program similiar to wget(1), but handles local file copy, ignores
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync# directories, and verifies file hashes.
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync#
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsyncimport os
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsyncimport sys
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsyncimport shutil
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsyncfrom urllib import splittype
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsyncfrom urllib2 import urlopen
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsyncimport hashlib
07557d07616212d7ba6e7ab3059e85cb14633775vboxsync
07557d07616212d7ba6e7ab3059e85cb14633775vboxsyncdef printIOError(e, txt):
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync """ Function to decode and print IOError type exception """
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync print "I/O Error: " + txt + ": "
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync try:
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync (code, message) = e
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync print str(message) + " (" + str(code) + ")"
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync except:
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync print str(e)
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsyncdef validate(file, hash):
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync algorithm, hashvalue = hash.split(':')
930b5f872e89407f445d4000d4e4aaecaa6a0998vboxsync
ed144bee38a6c8b914d3f027f1c118a2b0482d4fvboxsync # force migration away from sha1
if algorithm == "sha1":
algorithm = "sha256"
try:
m = hashlib.new(algorithm)
except ValueError:
return False
while True:
try:
block = file.read()
except IOError, err:
print str(err),
break
m.update(block)
if block == '':
break
return "%s:%s" % (algorithm, m.hexdigest())
def validate_container(filename, hash):
try:
file = open(filename, 'r')
except IOError as e:
printIOError(e, "Can't open file " + filename)
return False
return validate(file, hash)
def validate_payload(filename, hash):
import re
import gzip
import bz2
expr_bz = re.compile('.+\.bz2$', re.IGNORECASE)
expr_gz = re.compile('.+\.gz$', re.IGNORECASE)
expr_tgz = re.compile('.+\.tgz$', re.IGNORECASE)
try:
if expr_bz.match(filename):
file = bz2.BZ2File(filename, 'r')
elif expr_gz.match(filename):
file = gzip.GzipFile(filename, 'r')
elif expr_tgz.match(filename):
file = gzip.GzipFile(filename, 'r')
else:
return False
except IOError as e:
printIOError(e, "Can't open archive " + filename)
return False
return validate(file, hash)
def download(url, filename = None):
src = None
try:
src = urlopen(url)
except IOError as e:
printIOError(e, "Can't open url " + url)
return None
# 3xx, 4xx and 5xx (f|ht)tp codes designate unsuccessfull action
if 3 <= int(src.getcode()/100) <= 5:
print "Error code: " + str(src.getcode())
return None
if filename == None:
filename = src.geturl().split('/')[-1]
try:
dst = open(filename, 'wb');
except IOError as e:
printIOError(e, "Can't open file " + filename + " for writing")
src.close()
return None
while True:
block = src.read()
if block == '':
break;
dst.write(block)
src.close()
dst.close()
# return the name of the file that we downloaded the data to.
return filename
def download_paths(search, filename, url):
urls = list()
if filename != None:
tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
if tmp:
search += tmp.split(' ')
file = os.path.basename(filename)
urls = [ base + '/' + file for base in search ]
# filename should always be first
if filename in urls:
urls.remove(filename)
urls.insert(0, filename)
# command line url is a fallback, so it's last
if url != None and url not in urls:
urls.append(url)
return urls
def usage():
print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] [-s|--search (search-dir)] --url (url)" % (sys.argv[0].split('/')[-1])
sys.exit(1)
def main():
import getopt
# FLUSH STDOUT
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
file_arg = None
link_arg = False
hash_arg = None
url_arg = None
search_list = list()
try:
opts, args = getopt.getopt(sys.argv[1:], "f:h:ls:u:",
["file=", "link", "hash=", "search=", "url="])
except getopt.GetoptError, err:
print str(err)
usage()
for opt, arg in opts:
if opt in [ "-f", "--file" ]:
file_arg = arg
elif opt in [ "-l", "--link" ]:
link_arg = True
elif opt in [ "-h", "--hash" ]:
hash_arg = arg
elif opt in [ "-s", "--search" ]:
search_list.append(arg)
elif opt in [ "-u", "--url" ]:
url_arg = arg
else:
assert False, "unknown option"
if url_arg == None:
usage()
for url in download_paths(search_list, file_arg, url_arg):
print "Source %s..." % url,
scheme, path = splittype(url)
name = file_arg
if scheme in [ None, 'file' ]:
if os.path.exists(path) == False:
print "not found, skipping file copy"
continue
elif name != path:
if link_arg == False:
print "\n copying..."
shutil.copy2(path, name)
else:
print "\n linking..."
os.symlink(path, name)
else:
pass
elif scheme in [ 'http', 'https', 'ftp' ]:
print "\n downloading...",
name = download(url, file_arg)
if name == None:
print "failed"
continue
print "\n validating...",
if hash_arg == None:
print "skipping (no hash)"
sys.exit(0)
realhash = validate_container(name, hash_arg)
if realhash == hash_arg:
print "ok"
sys.exit(0)
else:
payloadhash = validate_payload(name, hash_arg)
if payloadhash == hash_arg:
print "ok"
sys.exit(0)
print "corruption detected"
print " expected: %s" % hash_arg
print " actual: %s" % realhash
print " payload: %s" % payloadhash
try:
os.remove(name)
except OSError:
pass
sys.exit(1)
if __name__ == "__main__":
main()