webutils.py revision cf22150eaeeb72431bf1cf65c309a431454fb22b
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync# -*- coding: utf-8 -*-
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync# $Id$
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync"""
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncCommon Web Utility Functions.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync"""
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync__copyright__ = \
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync"""
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncCopyright (C) 2012-2014 Oracle Corporation
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncThis file is part of VirtualBox Open Source Edition (OSE), as
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncavailable from http://www.virtualbox.org. This file is free software;
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncyou can redistribute it and/or modify it under the terms of the GNU
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncGeneral Public License (GPL) as published by the Free Software
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncFoundation, in version 2 as it comes in the "COPYING" file of the
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncVirtualBox OSE distribution. VirtualBox OSE is distributed in the
c785dbab313731d1f4662b4684c0808cc14703dbvboxsynchope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncThe contents of this file may alternatively be used under the terms
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncof the Common Development and Distribution License Version 1.0
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync(CDDL) only, as it comes in the "COPYING.CDDL" file of the
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncVirtualBox OSE distribution, in which case the provisions of the
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncCDDL are applicable instead of those of the GPL.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncYou may elect to license modified versions of this file under the
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncterms and conditions of either the GPL or the CDDL or both.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync"""
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync__version__ = "$Revision$"
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync# Standard Python imports.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncimport os;
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncimport shutil;
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncimport sys;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncimport unittest;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync# Validation Kit imports.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncfrom common import utils;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync# Python 3 hacks:
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncif sys.version_info[0] < 3:
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync from urllib import urlopen as urllib_urlopen;
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync from urllib2 import quote as urllib_quote;
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync from urllib import urlencode as urllib_urlencode;
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncelse:
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync from urllib.parse import quote as urllib_quote; # pylint: disable=F0401,E0611
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync from urllib.parse import urlencode as urllib_urlencode; # pylint: disable=F0401,E0611
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync from urllib.request import urlopen as urllib_urlopen; # pylint: disable=F0401,E0611
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncdef escapeElem(sText):
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync Escapes special character to HTML-safe sequences.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync sText = sText.replace('&', '&amp;')
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync sText = sText.replace('<', '&lt;')
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync return sText.replace('>', '&gt;')
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncdef escapeAttr(sText):
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync Escapes special character to HTML-safe sequences.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync sText = sText.replace('&', '&amp;')
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sText = sText.replace('<', '&lt;')
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync sText = sText.replace('>', '&gt;')
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync return sText.replace('"', '&quot;')
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncdef escapeElemToStr(oObject):
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync Stringifies the object and hands it to escapeElem.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync if utils.isString(oObject):
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync return escapeElem(oObject);
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync return escapeElem(str(oObject));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncdef escapeAttrToStr(oObject):
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync Stringifies the object and hands it to escapeAttr. May return unicode string.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync return escapeAttr(oObject);
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncdef escapeAttrJavaScriptStringDQ(sText):
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """ Escapes a javascript string that is to be emitted between double quotes. """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync if '"' not in sText:
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync chMin = min(sText);
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync if ord(chMin) >= 0x20:
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync return sText;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sRet = '';
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync for ch in sText:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync if ch == '"':
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sRet += '\\"';
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync elif ord(ch) >= 0x20:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sRet += ch;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync elif ch == '\n':
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sRet += '\\n';
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync elif ch == '\r':
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sRet += '\\r';
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync elif ch == '\t':
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sRet += '\\t';
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync else:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sRet += '\\x%02x' % (ch,);
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync return sRet;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncdef quoteUrl(sText):
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync See urllib.quote().
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync return urllib_quote(sText);
b83d9b1072dd8491c7ffe37830e8fd10f2dba561vboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncdef encodeUrlParams(dParams):
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync See urllib.urlencode().
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync return urllib_urlencode(dParams, doseq=True)
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncdef hasSchema(sUrl):
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync Checks if the URL has a schema (e.g. http://) or is file/server relative.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync Returns True if schema is present, False if not.
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync """
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync iColon = sUrl.find(':');
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync if iColon > 0:
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync sSchema = sUrl[0:iColon];
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync if len(sSchema) >= 2 and len(sSchema) < 16 and sSchema.islower() and sSchema.isalpha():
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync return True;
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync return False;
c785dbab313731d1f4662b4684c0808cc14703dbvboxsync
c785dbab313731d1f4662b4684c0808cc14703dbvboxsyncdef getFilename(sUrl):
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync Extracts the filename from the URL.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync ## @TODO This isn't entirely correct. Use the urlparser instead!
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sFilename = os.path.basename(sUrl.replace('/', os.path.sep));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync return sFilename;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncdef downloadFile(sUrlFile, sDstFile, sLocalPrefix, fnLog, fnError = None, fNoProxies=True):
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync Downloads the given file if an URL is given, otherwise assume it's
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync something on the build share and copy it from there.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync Raises no exceptions, returns log + success indicator instead.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync Note! This method may use proxies configured on the system and the
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync http_proxy, ftp_proxy, no_proxy environment variables.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync @todo Fixed build burn here. Please set default value for fNoProxies
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync to appropriate one.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync """
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync if fnError is None:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync fnError = fnLog;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync if sUrlFile.startswith('http://') \
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync or sUrlFile.startswith('https://') \
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync or sUrlFile.startswith('ftp://'):
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync # Download the file.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync fnLog('Downloading "%s" to "%s"...' % (sUrlFile, sDstFile));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync try:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync ## @todo We get 404.html content instead of exceptions here, which is confusing and should be addressed.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync if fNoProxies:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync oSrc = urllib_urlopen(sUrlFile);
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync else:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync oSrc = urllib_urlopen(sUrlFile, proxies = dict());
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync oDst = utils.openNoInherit(sDstFile, 'wb');
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync oDst.write(oSrc.read());
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync oDst.close();
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync oSrc.close();
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync except Exception, oXcpt:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync fnError('Error downloading "%s" to "%s": %s' % (sUrlFile, sDstFile, oXcpt));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync return False;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync else:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync # Assumes file from the build share.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync sSrcPath = os.path.join(sLocalPrefix, sUrlFile);
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync fnLog('Copying "%s" to "%s"...' % (sSrcPath, sDstFile));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync try:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync shutil.copyfile(sSrcPath, sDstFile);
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync except Exception, oXcpt:
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync fnError('Error copying "%s" to "%s": %s' % (sSrcPath, sDstFile, oXcpt));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync return False;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync return True;
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync#
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync# Unit testing.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync#
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync# pylint: disable=C0111
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncclass CommonUtilsTestCase(unittest.TestCase):
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync def testHasSchema(self):
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync self.assertTrue(hasSchema('http://www.oracle.com/'));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync self.assertTrue(hasSchema('https://virtualbox.com/'));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync self.assertFalse(hasSchema('://virtualbox.com/'));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync self.assertFalse(hasSchema('/usr/bin'));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync self.assertFalse(hasSchema('usr/bin'));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync self.assertFalse(hasSchema('bin'));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync self.assertFalse(hasSchema('C:\\WINNT'));
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsyncif __name__ == '__main__':
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync unittest.main();
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync # not reached.
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync
3c3a5ab35783f4d31cb5d3a15db9daadeb804daavboxsync