userland-fetch revision 137
199767f8919635c4928607450d9e0abb932109ceToomas Soome#!/usr/bin/python2.6
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome# CDDL HEADER START
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome# The contents of this file are subject to the terms of the
199767f8919635c4928607450d9e0abb932109ceToomas Soome# Common Development and Distribution License (the "License").
199767f8919635c4928607450d9e0abb932109ceToomas Soome# You may not use this file except in compliance with the License.
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
199767f8919635c4928607450d9e0abb932109ceToomas Soome# or http://www.opensolaris.org/os/licensing.
199767f8919635c4928607450d9e0abb932109ceToomas Soome# See the License for the specific language governing permissions
199767f8919635c4928607450d9e0abb932109ceToomas Soome# and limitations under the License.
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome# When distributing Covered Code, include this CDDL HEADER in each
199767f8919635c4928607450d9e0abb932109ceToomas Soome# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
199767f8919635c4928607450d9e0abb932109ceToomas Soome# If applicable, add the following below this CDDL HEADER, with the
199767f8919635c4928607450d9e0abb932109ceToomas Soome# fields enclosed by brackets "[]" replaced with your own identifying
199767f8919635c4928607450d9e0abb932109ceToomas Soome# information: Portions Copyright [yyyy] [name of copyright owner]
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome# CDDL HEADER END
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome# Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved.
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome# fetch.py - a file download utility
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome# A simple program similiar to wget(1), but handles local file copy, ignores
199767f8919635c4928607450d9e0abb932109ceToomas Soome# directories, and verifies file hashes.
199767f8919635c4928607450d9e0abb932109ceToomas Soome#
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomeimport os
199767f8919635c4928607450d9e0abb932109ceToomas Soomeimport sys
199767f8919635c4928607450d9e0abb932109ceToomas Soomeimport shutil
199767f8919635c4928607450d9e0abb932109ceToomas Soomefrom urllib import splittype, urlopen
199767f8919635c4928607450d9e0abb932109ceToomas Soomeimport hashlib
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomedef printIOError(e, txt):
199767f8919635c4928607450d9e0abb932109ceToomas Soome """ Function to decode and print IOError type exception """
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "I/O Error: " + txt + ": "
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome (code, message) = e
199767f8919635c4928607450d9e0abb932109ceToomas Soome print str(message) + " (" + str(code) + ")"
199767f8919635c4928607450d9e0abb932109ceToomas Soome except:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print str(e)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomedef validate(file, hash):
199767f8919635c4928607450d9e0abb932109ceToomas Soome algorithm, hashvalue = hash.split(':')
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome m = hashlib.new(algorithm)
199767f8919635c4928607450d9e0abb932109ceToomas Soome except ValueError:
199767f8919635c4928607450d9e0abb932109ceToomas Soome return False
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome while True:
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome block = file.read()
199767f8919635c4928607450d9e0abb932109ceToomas Soome except IOError, err:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print str(err),
199767f8919635c4928607450d9e0abb932109ceToomas Soome break
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome m.update(block)
199767f8919635c4928607450d9e0abb932109ceToomas Soome if block == '':
199767f8919635c4928607450d9e0abb932109ceToomas Soome break
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome return "%s:%s" % (algorithm, m.hexdigest())
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomedef validate_container(filename, hash):
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome file = open(filename, 'r')
199767f8919635c4928607450d9e0abb932109ceToomas Soome except IOError as e:
199767f8919635c4928607450d9e0abb932109ceToomas Soome printIOError(e, "Can't open file " + filename)
199767f8919635c4928607450d9e0abb932109ceToomas Soome return False
199767f8919635c4928607450d9e0abb932109ceToomas Soome return validate(file, hash)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomedef validate_payload(filename, hash):
199767f8919635c4928607450d9e0abb932109ceToomas Soome import re
199767f8919635c4928607450d9e0abb932109ceToomas Soome import gzip
199767f8919635c4928607450d9e0abb932109ceToomas Soome import bz2
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome expr_bz = re.compile('.+\.bz2$', re.IGNORECASE)
199767f8919635c4928607450d9e0abb932109ceToomas Soome expr_gz = re.compile('.+\.gz$', re.IGNORECASE)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome if expr_bz.match(filename):
199767f8919635c4928607450d9e0abb932109ceToomas Soome file = bz2.BZ2File(filename, 'r')
199767f8919635c4928607450d9e0abb932109ceToomas Soome elif expr_gz.match(filename):
199767f8919635c4928607450d9e0abb932109ceToomas Soome file = gzip.GzipFile(filename, 'r')
199767f8919635c4928607450d9e0abb932109ceToomas Soome else:
199767f8919635c4928607450d9e0abb932109ceToomas Soome return False
199767f8919635c4928607450d9e0abb932109ceToomas Soome except IOError as e:
199767f8919635c4928607450d9e0abb932109ceToomas Soome printIOError(e, "Can't open archive " + filename)
199767f8919635c4928607450d9e0abb932109ceToomas Soome return False
199767f8919635c4928607450d9e0abb932109ceToomas Soome return validate(file, hash)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomedef download(url, filename = None):
199767f8919635c4928607450d9e0abb932109ceToomas Soome src = None
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome src = urlopen(url)
199767f8919635c4928607450d9e0abb932109ceToomas Soome except IOError as e:
199767f8919635c4928607450d9e0abb932109ceToomas Soome printIOError(e, "Can't open url " + url)
199767f8919635c4928607450d9e0abb932109ceToomas Soome return None
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome if filename == None:
199767f8919635c4928607450d9e0abb932109ceToomas Soome filename = src.geturl().split('/')[-1]
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome dst = open(filename, 'wb');
199767f8919635c4928607450d9e0abb932109ceToomas Soome except IOError as e:
199767f8919635c4928607450d9e0abb932109ceToomas Soome printIOError(e, "Can't open file " + filename + " for writing")
199767f8919635c4928607450d9e0abb932109ceToomas Soome src.close()
199767f8919635c4928607450d9e0abb932109ceToomas Soome return None
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome while True:
199767f8919635c4928607450d9e0abb932109ceToomas Soome block = src.read()
199767f8919635c4928607450d9e0abb932109ceToomas Soome if block == '':
199767f8919635c4928607450d9e0abb932109ceToomas Soome break;
199767f8919635c4928607450d9e0abb932109ceToomas Soome dst.write(block)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome src.close()
199767f8919635c4928607450d9e0abb932109ceToomas Soome dst.close()
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome # return the name of the file that we downloaded the data to.
199767f8919635c4928607450d9e0abb932109ceToomas Soome return filename
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomedef download_paths(search, filename, url):
199767f8919635c4928607450d9e0abb932109ceToomas Soome urls = list()
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome if filename != None:
199767f8919635c4928607450d9e0abb932109ceToomas Soome tmp = os.getenv('DOWNLOAD_SEARCH_PATH')
199767f8919635c4928607450d9e0abb932109ceToomas Soome if tmp:
199767f8919635c4928607450d9e0abb932109ceToomas Soome search += tmp.split(' ')
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome file = os.path.basename(filename)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome urls = [ base + '/' + file for base in search ]
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome # filename should always be first
199767f8919635c4928607450d9e0abb932109ceToomas Soome if filename in urls:
199767f8919635c4928607450d9e0abb932109ceToomas Soome urls.remove(filename)
199767f8919635c4928607450d9e0abb932109ceToomas Soome urls.insert(0, filename)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome # command line url is a fallback, so it's last
199767f8919635c4928607450d9e0abb932109ceToomas Soome if url != None and url not in urls:
199767f8919635c4928607450d9e0abb932109ceToomas Soome urls.append(url)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome return urls
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomedef usage():
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "Usage: %s [-f|--file (file)] [-l|--link] [-h|--hash (hash)] [-s|--search (search-dir)] --url (url)" % (sys.argv[0].split('/')[-1])
199767f8919635c4928607450d9e0abb932109ceToomas Soome sys.exit(1)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomedef main():
199767f8919635c4928607450d9e0abb932109ceToomas Soome import getopt
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome # FLUSH STDOUT
199767f8919635c4928607450d9e0abb932109ceToomas Soome sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome file_arg = None
199767f8919635c4928607450d9e0abb932109ceToomas Soome link_arg = False
199767f8919635c4928607450d9e0abb932109ceToomas Soome hash_arg = None
199767f8919635c4928607450d9e0abb932109ceToomas Soome url_arg = None
199767f8919635c4928607450d9e0abb932109ceToomas Soome search_list = list()
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome opts, args = getopt.getopt(sys.argv[1:], "f:h:ls:u:",
199767f8919635c4928607450d9e0abb932109ceToomas Soome ["file=", "link", "hash=", "search=", "url="])
199767f8919635c4928607450d9e0abb932109ceToomas Soome except getopt.GetoptError, err:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print str(err)
199767f8919635c4928607450d9e0abb932109ceToomas Soome usage()
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome for opt, arg in opts:
199767f8919635c4928607450d9e0abb932109ceToomas Soome if opt in [ "-f", "--file" ]:
199767f8919635c4928607450d9e0abb932109ceToomas Soome file_arg = arg
199767f8919635c4928607450d9e0abb932109ceToomas Soome elif opt in [ "-l", "--link" ]:
199767f8919635c4928607450d9e0abb932109ceToomas Soome link_arg = True
199767f8919635c4928607450d9e0abb932109ceToomas Soome elif opt in [ "-h", "--hash" ]:
199767f8919635c4928607450d9e0abb932109ceToomas Soome hash_arg = arg
199767f8919635c4928607450d9e0abb932109ceToomas Soome elif opt in [ "-s", "--search" ]:
199767f8919635c4928607450d9e0abb932109ceToomas Soome search_list.append(arg)
199767f8919635c4928607450d9e0abb932109ceToomas Soome elif opt in [ "-u", "--url" ]:
199767f8919635c4928607450d9e0abb932109ceToomas Soome url_arg = arg
199767f8919635c4928607450d9e0abb932109ceToomas Soome else:
199767f8919635c4928607450d9e0abb932109ceToomas Soome assert False, "unknown option"
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome if url_arg == None:
199767f8919635c4928607450d9e0abb932109ceToomas Soome usage()
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome for url in download_paths(search_list, file_arg, url_arg):
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "Source %s..." % url,
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome scheme, path = splittype(url)
199767f8919635c4928607450d9e0abb932109ceToomas Soome name = file_arg
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome if scheme in [ None, 'file' ]:
199767f8919635c4928607450d9e0abb932109ceToomas Soome if os.path.exists(path) == False:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "not found, skipping file copy"
199767f8919635c4928607450d9e0abb932109ceToomas Soome continue
199767f8919635c4928607450d9e0abb932109ceToomas Soome elif name != path:
199767f8919635c4928607450d9e0abb932109ceToomas Soome if link_arg == False:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "\n copying..."
199767f8919635c4928607450d9e0abb932109ceToomas Soome shutil.copy2(path, name)
199767f8919635c4928607450d9e0abb932109ceToomas Soome else:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "\n linking..."
199767f8919635c4928607450d9e0abb932109ceToomas Soome os.symlink(path, name)
199767f8919635c4928607450d9e0abb932109ceToomas Soome else:
199767f8919635c4928607450d9e0abb932109ceToomas Soome pass
199767f8919635c4928607450d9e0abb932109ceToomas Soome elif scheme in [ 'http', 'https', 'ftp' ]:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "\n downloading...",
199767f8919635c4928607450d9e0abb932109ceToomas Soome name = download(url, file_arg)
199767f8919635c4928607450d9e0abb932109ceToomas Soome if name == None:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "failed"
199767f8919635c4928607450d9e0abb932109ceToomas Soome continue
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "\n validating...",
199767f8919635c4928607450d9e0abb932109ceToomas Soome if hash_arg == None:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "skipping (no hash)"
199767f8919635c4928607450d9e0abb932109ceToomas Soome sys.exit(0)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome realhash = validate_container(name, hash_arg)
199767f8919635c4928607450d9e0abb932109ceToomas Soome if realhash == hash_arg:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "ok"
199767f8919635c4928607450d9e0abb932109ceToomas Soome sys.exit(0)
199767f8919635c4928607450d9e0abb932109ceToomas Soome else:
199767f8919635c4928607450d9e0abb932109ceToomas Soome payloadhash = validate_payload(name, hash_arg)
199767f8919635c4928607450d9e0abb932109ceToomas Soome if payloadhash == hash_arg:
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "ok"
199767f8919635c4928607450d9e0abb932109ceToomas Soome sys.exit(0)
199767f8919635c4928607450d9e0abb932109ceToomas Soome print "corruption detected"
199767f8919635c4928607450d9e0abb932109ceToomas Soome print " expected: %s" % hash_arg
199767f8919635c4928607450d9e0abb932109ceToomas Soome print " actual: %s" % realhash
199767f8919635c4928607450d9e0abb932109ceToomas Soome print " payload: %s" % payloadhash
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome try:
199767f8919635c4928607450d9e0abb932109ceToomas Soome os.remove(name)
199767f8919635c4928607450d9e0abb932109ceToomas Soome except OSError:
199767f8919635c4928607450d9e0abb932109ceToomas Soome pass
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soome sys.exit(1)
199767f8919635c4928607450d9e0abb932109ceToomas Soome
199767f8919635c4928607450d9e0abb932109ceToomas Soomeif __name__ == "__main__":
199767f8919635c4928607450d9e0abb932109ceToomas Soome main()
199767f8919635c4928607450d9e0abb932109ceToomas Soome