depot.py revision 612
1516N/A#!/usr/bin/python2.4
20N/A#
20N/A# CDDL HEADER START
20N/A#
20N/A# The contents of this file are subject to the terms of the
20N/A# Common Development and Distribution License (the "License").
20N/A# You may not use this file except in compliance with the License.
20N/A#
20N/A# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
20N/A# or http://www.opensolaris.org/os/licensing.
20N/A# See the License for the specific language governing permissions
20N/A# and limitations under the License.
20N/A#
20N/A# When distributing Covered Code, include this CDDL HEADER in each
20N/A# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
20N/A# If applicable, add the following below this CDDL HEADER, with the
20N/A# fields enclosed by brackets "[]" replaced with your own identifying
20N/A# information: Portions Copyright [yyyy] [name of copyright owner]
20N/A#
20N/A# CDDL HEADER END
20N/A#
3053N/A# Copyright 2008 Sun Microsystems, Inc. All rights reserved.
20N/A# Use is subject to license terms.
20N/A#
3143N/A
3143N/A# pkg.depotd - package repository daemon
22N/A
0N/A# XXX The prototype pkg.depotd combines both the version management server that
50N/A# answers to pkgsend(1) sessions and the HTTP file server that answers to the
50N/A# various GET operations that a pkg(1) client makes. This split is expected to
50N/A# be made more explicit, by constraining the pkg(1) operations such that they
50N/A# can be served as a typical HTTP/HTTPS session. Thus, pkg.depotd will reduce
50N/A# to a special purpose HTTP/HTTPS server explicitly for the version management
50N/A# operations, and must manipulate the various state files--catalogs, in
50N/A# particular--such that the pkg(1) pull client can operately accurately with
50N/A# only a basic HTTP/HTTPS server in place.
50N/A
50N/A# XXX We should support simple "last-modified" operations via HEAD queries.
50N/A
50N/A# XXX Although we pushed the evaluation of next-version, etc. to the pull
50N/A# client, we should probably provide a query API to do same on the server, for
50N/A# dumb clients (like a notification service).
589N/A
589N/A# The default authority for the depot.
965N/AAUTH_DEFAULT = "opensolaris.org"
965N/A# The default repository path.
965N/AREPO_PATH_DEFAULT = "/var/pkg/repo"
965N/A# The default path for static and other web content.
965N/ACONTENT_PATH_DEFAULT = "/usr/share/lib/pkg"
2951N/A# The default port to serve data from.
1836N/APORT_DEFAULT = 80
1836N/A# The minimum number of threads allowed.
382N/ATHREADS_MIN = 1
812N/A# The default number of threads to start.
382N/ATHREADS_DEFAULT = 10
382N/A# The maximum number of threads that can be started.
382N/ATHREADS_MAX = 100
1963N/A# The default server socket timeout in seconds. We want this to be longer than
382N/A# the normal default of 10 seconds to accommodate clients with poor quality
1963N/A# connections.
382N/ASOCKET_TIMEOUT_DEFAULT = 60
382N/A# Whether modify operations should be allowed.
382N/AREADONLY_DEFAULT = False
382N/A# Whether the repository catalog should be rebuilt on startup.
382N/AREBUILD_DEFAULT = False
26N/A# Whether the indexes should be rebuilt
689N/AREINDEX_DEFAULT = False
689N/A# Not in mirror mode by default
466N/AMIRROR_DEFAULT = False
0N/A
468N/Aimport getopt
812N/Aimport logging
812N/Aimport os
52N/Aimport os.path
812N/Aimport sys
451N/Aimport urlparse
0N/A
382N/Atry:
382N/A import cherrypy
382N/A version = cherrypy.__version__.split('.')
452N/A if map(int, version) < [3, 1, 0]:
382N/A raise ImportError
452N/A elif map(int, version) >= [3, 2, 0]:
382N/A raise ImportError
382N/Aexcept ImportError:
3143N/A print """cherrypy 3.1.0 or greater (but less than 3.2.0) is """ \
3143N/A """required to use this program."""
382N/A sys.exit(2)
22N/A
1836N/Aimport pkg.server.face as face
2507N/Aimport pkg.server.config as config
1836N/Aimport pkg.server.depot as depot
1836N/Aimport pkg.server.repository as repo
2962N/Aimport pkg.server.repositoryconfig as rc
2962N/Afrom pkg.misc import port_available, msg, emsg
2962N/A
1431N/Aclass LogSink(object):
1968N/A """This is a dummy object that we can use to discard log entries
873N/A without relying on non-portable interfaces such as /dev/null."""
812N/A
1431N/A def write(self, *args, **kwargs):
873N/A """Discard the bits."""
1431N/A pass
466N/A
1431N/A def flush(self, *args, **kwargs):
466N/A """Discard the bits."""
466N/A pass
466N/A
23N/Adef usage(text):
466N/A if text:
466N/A emsg(text)
466N/A
466N/A print """\
466N/AUsage: /usr/lib/pkg.depotd [-d repo_dir] [-p port] [-s threads]
466N/A [-t socket_timeout] [--content-root] [--log-access dest]
466N/A [--log-errors dest] [--mirror] [--proxy-base url] [--readonly]
466N/A [--rebuild]
1431N/A
1633N/A --content-root The file system path to the directory containing the
1633N/A the static and other web content used by the depot's
1633N/A browser user interface. The default value is
1633N/A '/usr/share/lib/pkg'.
466N/A --log-access The destination for any access related information
466N/A logged by the depot process. Possible values are:
466N/A stderr, stdout, none, or an absolute pathname. The
1633N/A default value is stdout if stdout is a tty; otherwise
1633N/A the default value is none.
1633N/A --log-errors The destination for any errors or other information
1633N/A logged by the depot process. Possible values are:
1633N/A stderr, stdout, none, or an absolute pathname. The
1633N/A default value is stderr.
3143N/A --mirror Package mirror mode; publishing and metadata operations
2230N/A disallowed. Cannot be used with --readonly or
1968N/A --rebuild.
1633N/A --proxy-base The url to use as the base for generating internal
2515N/A redirects and content.
2816N/A --readonly Read-only operation; modifying operations disallowed.
2816N/A Cannot be used with --mirror or --rebuild.
1937N/A --rebuild Re-build the catalog from pkgs in depot. Cannot be
382N/A used with --mirror or --readonly.
2230N/A"""
2230N/A sys.exit(2)
2230N/A
2230N/Aclass OptionError(Exception):
2028N/A """Option exception. """
1968N/A
1968N/A def __init__(self, *args):
2028N/A Exception.__init__(self, *args)
1968N/A
1968N/Aif __name__ == "__main__":
1968N/A
2028N/A port = PORT_DEFAULT
2028N/A threads = THREADS_DEFAULT
2028N/A socket_timeout = SOCKET_TIMEOUT_DEFAULT
1968N/A readonly = READONLY_DEFAULT
1968N/A rebuild = REBUILD_DEFAULT
1968N/A reindex = REINDEX_DEFAULT
1968N/A proxy_base = None
1968N/A mirror = MIRROR_DEFAULT
1968N/A
589N/A if "PKG_REPO" in os.environ:
589N/A repo_path = os.environ["PKG_REPO"]
589N/A else:
589N/A repo_path = REPO_PATH_DEFAULT
1431N/A
1431N/A try:
1431N/A content_root = os.environ["PKG_DEPOT_CONTENT"]
1431N/A except KeyError:
1431N/A try:
858N/A content_root = os.path.join(os.environ['PKG_HOME'],
1633N/A 'share/lib/pkg')
2962N/A except KeyError:
3053N/A content_root = CONTENT_PATH_DEFAULT
2515N/A
2515N/A # By default, if the destination for a particular log type is not
466N/A # specified, this is where we will send the output.
466N/A log_routes = {
466N/A "access": "none",
466N/A "errors": "stderr"
466N/A }
466N/A log_opts = ["--log-%s" % log_type for log_type in log_routes]
466N/A
466N/A # If stdout is a tty, then send access output there by default instead
466N/A # of discarding it.
589N/A if os.isatty(sys.stdout.fileno()):
589N/A log_routes["access"] = "stdout"
589N/A
1191N/A opt = None
1191N/A try:
1191N/A long_opts = ["content-root=", "mirror", "proxy-base=",
1191N/A "readonly", "rebuild", "refresh-index"]
1191N/A for opt in log_opts:
2816N/A long_opts.append("%s=" % opt.lstrip('--'))
2816N/A opts, pargs = getopt.getopt(sys.argv[1:], "d:np:s:t:",
589N/A long_opts)
589N/A for opt, arg in opts:
589N/A if opt == "-n":
589N/A sys.exit(0)
812N/A elif opt == "-d":
812N/A repo_path = arg
812N/A elif opt == "-p":
812N/A port = int(arg)
812N/A elif opt == "-s":
812N/A threads = int(arg)
812N/A if threads < THREADS_MIN:
1968N/A raise OptionError, \
1968N/A "minimum value is %d" % THREADS_MIN
1968N/A if threads > THREADS_MAX:
812N/A raise OptionError, \
812N/A "maximum value is %d" % THREADS_MAX
812N/A elif opt == "-t":
812N/A socket_timeout = int(arg)
1475N/A elif opt == "--content-root":
1475N/A if arg == "":
1475N/A raise OptionError, "You must specify " \
1475N/A "a directory path."
975N/A content_root = arg
975N/A elif opt in log_opts:
975N/A if arg is None or arg == "":
975N/A raise OptionError, \
1633N/A "You must specify a log " \
1633N/A "destination."
1633N/A log_routes[opt.lstrip("--log-")] = arg
1633N/A elif opt == "--mirror":
2028N/A mirror = True
1633N/A elif opt == "--proxy-base":
3143N/A # Attempt to decompose the url provided into
1633N/A # its base parts. This is done so we can
14N/A # remove any scheme information since we
382N/A # don't need it.
429N/A scheme, netloc, path, params, query, \
14N/A fragment = urlparse.urlparse(arg,
404N/A allow_fragments=0)
404N/A
30N/A # Rebuild the url without the scheme and
382N/A # remove the leading // urlunparse adds.
30N/A proxy_base = urlparse.urlunparse(("", netloc,
791N/A path, params, query, fragment)
2728N/A ).lstrip("//")
2728N/A elif opt == "--readonly":
689N/A readonly = True
1968N/A elif opt == "--rebuild":
1968N/A rebuild = True
1968N/A elif opt == "--refresh-index":
1968N/A # Note: This argument is for internal use
1191N/A # only. It's used when pkg.depotd is reexecing
258N/A # itself and needs to know that's the case.
1968N/A # This flag is purposefully omitted in usage.
2816N/A # The supported way to forcefully reindex is to
382N/A # kill any pkg.depot using that directory,
1968N/A # remove the index directory, and restart the
30N/A # pkg.depot process. The index will be rebuilt
589N/A # automatically on startup.
589N/A reindex = True
1968N/A except getopt.GetoptError, e:
589N/A usage("pkg.depotd: %s" % e.msg)
589N/A except OptionError, e:
589N/A usage("pkg.depotd: option: %s -- %s" % (opt, e))
589N/A except (ArithmeticError, ValueError):
1968N/A usage("pkg.depotd: illegal option value: %s specified " \
589N/A "for option: %s" % (arg, opt))
1968N/A
466N/A if rebuild and reindex:
466N/A usage("--refresh-index cannot be used with --rebuild")
2230N/A if rebuild and (readonly or mirror):
1968N/A usage("--readonly and --mirror cannot be used with --rebuild")
1968N/A if reindex and (readonly or mirror):
1431N/A usage("--readonly and --mirror cannot be used with " \
1968N/A "--refresh-index")
1968N/A
54N/A # If the program is going to reindex, the port is irrelevant since
1968N/A # the program will not bind to a port.
1968N/A if not reindex:
2515N/A available, msg = port_available(None, port)
2816N/A if not available:
2816N/A print "pkg.depotd: unable to bind to the specified " \
2816N/A "port: %d. Reason: %s" % (port, msg)
2816N/A sys.exit(1)
1475N/A else:
2230N/A # Not applicable for reindexing operations.
466N/A content_root = None
1633N/A
1633N/A scfg = config.SvrConfig(repo_path, content_root, AUTH_DEFAULT)
135N/A
2230N/A if rebuild:
2230N/A scfg.destroy_catalog()
2230N/A
135N/A if readonly:
135N/A scfg.set_read_only()
1968N/A
135N/A if mirror:
1968N/A scfg.set_mirror()
382N/A
382N/A try:
382N/A scfg.init_dirs()
382N/A except (RuntimeError, EnvironmentError), e:
382N/A print "pkg.depotd: an error occurred while trying to " \
382N/A "initialize the depot repository directory " \
382N/A "structures:\n%s" % e
382N/A sys.exit(1)
1968N/A
382N/A # Setup our global configuration.
1968N/A # Global cherrypy configuration
1542N/A gconf = {
1542N/A "environment": "production",
1968N/A "checker.on": True,
1968N/A "log.screen": False,
812N/A "server.socket_host": "0.0.0.0",
1968N/A "server.socket_port": port,
589N/A "server.thread_pool": threads,
1968N/A "server.socket_timeout": socket_timeout,
858N/A "tools.log_headers.on": True
858N/A }
1968N/A
858N/A log_type_map = {
858N/A "errors": {
858N/A "param": "log.error_file",
858N/A "attr": "error_log"
858N/A },
858N/A "access": {
858N/A "param": "log.access_file",
1968N/A "attr": "access_log"
2962N/A }
2962N/A }
2962N/A
2962N/A for log_type in log_type_map:
2962N/A dest = log_routes[log_type]
2962N/A if dest in ("stdout", "stderr", "none"):
2962N/A if dest == "none":
2962N/A h = logging.StreamHandler(LogSink())
2962N/A else:
2962N/A h = logging.StreamHandler(eval("sys.%s" % \
1431N/A dest))
1431N/A
1431N/A h.setLevel(logging.DEBUG)
1431N/A h.setFormatter(cherrypy._cplogging.logfmt)
1431N/A log_obj = eval("cherrypy.log.%s" % \
1431N/A log_type_map[log_type]["attr"])
1431N/A log_obj.addHandler(h)
1431N/A # Since we've replaced cherrypy's log handler with our
1431N/A # own, we don't want the output directed to a file.
1431N/A dest = ""
1431N/A
1431N/A gconf[log_type_map[log_type]["param"]] = dest
1431N/A
1431N/A cherrypy.config.update(gconf)
1431N/A
1431N/A # Now that our logging, etc. has been setup, it's safe to perform any
1431N/A # remaining preparation.
1431N/A if reindex:
1968N/A scfg.acquire_catalog(rebuild=False)
1542N/A scfg.catalog.run_update_index()
1542N/A sys.exit(0)
2515N/A
2515N/A # Now build our site configuration.
1968N/A conf = {
1968N/A "/": {
1968N/A # We have to override cherrypy's default response_class so that
1633N/A # we have access to the write() callable to stream data
1633N/A # directly to the client.
589N/A "wsgi.response_class": depot.DepotResponse,
1968N/A },
1902N/A "/robots.txt": {
1968N/A "tools.staticfile.on": True,
1968N/A "tools.staticfile.filename": os.path.join(scfg.web_static_root,
1968N/A "robots.txt")
1191N/A },
2816N/A "/static": {
2816N/A "tools.staticdir.on": True,
2816N/A "tools.staticdir.root": scfg.web_static_root,
1191N/A "tools.staticdir.dir": ""
1191N/A }
1191N/A }
1191N/A
2816N/A if proxy_base:
2816N/A # This changes the base URL for our server, and is primarily
2816N/A # intended to allow our depot process to operate behind Apache
2816N/A # or some other webserver process.
2816N/A #
589N/A # Visit the following URL for more information:
589N/A # http://cherrypy.org/wiki/BuiltinTools#tools.proxy
589N/A proxy_conf = {
589N/A "tools.proxy.on": True,
589N/A "tools.proxy.local": "",
589N/A "tools.proxy.base": proxy_base
589N/A }
765N/A
765N/A if "/" not in conf:
765N/A conf["/"] = {}
765N/A
765N/A # Now merge or add our proxy configuration information into the
765N/A # existing configuration.
765N/A for entry in proxy_conf:
589N/A conf["/"][entry] = proxy_conf[entry]
765N/A
765N/A scfg.acquire_in_flight()
765N/A scfg.acquire_catalog()
765N/A
765N/A try:
765N/A root = cherrypy.Application(repo.Repository(scfg))
765N/A except rc.InvalidAttributeValueError, e:
1968N/A emsg("pkg.depotd: repository.conf error: %s" % e)
1968N/A sys.exit(1)
1968N/A
135N/A try:
1968N/A cherrypy.quickstart(root, config=conf)
157N/A except:
382N/A usage("pkg.depotd: unknown error starting depot, illegal " \
429N/A "option value specified?")
429N/A
2028N/A