depot.py revision 451
409N/A#!/usr/bin/python2.4
20N/A#
20N/A# CDDL HEADER START
20N/A#
20N/A# The contents of this file are subject to the terms of the
20N/A# Common Development and Distribution License (the "License").
20N/A# You may not use this file except in compliance with the License.
20N/A#
20N/A# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
20N/A# or http://www.opensolaris.org/os/licensing.
20N/A# See the License for the specific language governing permissions
20N/A# and limitations under the License.
20N/A#
20N/A# When distributing Covered Code, include this CDDL HEADER in each
20N/A# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
20N/A# If applicable, add the following below this CDDL HEADER, with the
20N/A# fields enclosed by brackets "[]" replaced with your own identifying
20N/A# information: Portions Copyright [yyyy] [name of copyright owner]
20N/A#
20N/A# CDDL HEADER END
20N/A#
260N/A# Copyright 2008 Sun Microsystems, Inc. All rights reserved.
20N/A# Use is subject to license terms.
20N/A#
20N/A
22N/A# pkg.depotd - package repository daemon
0N/A
50N/A# XXX The prototype pkg.depotd combines both the version management server that
50N/A# answers to pkgsend(1) sessions and the HTTP file server that answers to the
50N/A# various GET operations that a pkg(1) client makes. This split is expected to
50N/A# be made more explicit, by constraining the pkg(1) operations such that they
50N/A# can be served as a typical HTTP/HTTPS session. Thus, pkg.depotd will reduce
50N/A# to a special purpose HTTP/HTTPS server explicitly for the version management
50N/A# operations, and must manipulate the various state files--catalogs, in
50N/A# particular--such that the pkg(1) pull client can operately accurately with
50N/A# only a basic HTTP/HTTPS server in place.
50N/A
50N/A# XXX We should support simple "last-modified" operations via HEAD queries.
50N/A
50N/A# XXX Although we pushed the evaluation of next-version, etc. to the pull
50N/A# client, we should probably provide a query API to do same on the server, for
50N/A# dumb clients (like a notification service).
50N/A
382N/A# The default authority for the depot.
382N/AAUTH_DEFAULT = "opensolaris.org"
382N/A# The default repository path.
382N/AREPO_PATH_DEFAULT = "/var/pkg/repo"
382N/A# The default port to serve data from.
382N/APORT_DEFAULT = 80
382N/A# The minimum number of threads allowed.
382N/ATHREADS_MIN = 1
382N/A# The default number of threads to start.
382N/ATHREADS_DEFAULT = 10
382N/A# The maximum number of threads that can be started.
382N/ATHREADS_MAX = 100
382N/A# The default server socket timeout in seconds. We want this to be longer than
382N/A# the normal default of 10 seconds to accommodate clients with poor quality
382N/A# connections.
382N/ASOCKET_TIMEOUT_DEFAULT = 60
382N/A# Whether modify operations should be allowed.
382N/AREADONLY_DEFAULT = False
382N/A# Whether the repository catalog should be rebuilt on startup.
382N/AREBUILD_DEFAULT = False
429N/A# Whether the indexes should be rebuilt
429N/AREINDEX_DEFAULT = False
382N/A
26N/Aimport getopt
0N/Aimport os
52N/Aimport sys
451N/Aimport urlparse
0N/A
382N/Atry:
382N/A import cherrypy
382N/A version = cherrypy.__version__.split('.')
382N/A if map(int, version) < [3, 0, 3]:
382N/A raise ImportError
382N/A elif map(int, version) >= [3, 1, 0]:
382N/A raise ImportError
382N/Aexcept ImportError:
382N/A print """cherrypy 3.0.3 or greater (but less than 3.1.0) is """ \
382N/A """required to use this program."""
382N/A sys.exit(2)
22N/A
114N/Aimport pkg.server.face as face
26N/Aimport pkg.server.config as config
382N/Aimport pkg.server.depot as depot
382N/Aimport pkg.server.repository as repo
428N/Aimport pkg.server.repositoryconfig as rc
428N/Afrom pkg.misc import port_available, emsg
23N/A
26N/Adef usage():
26N/A print """\
451N/AUsage: /usr/lib/pkg.depotd [--readonly] [--rebuild] [--proxy-base url]
451N/A [-d repo_dir] [-p port] [-s threads] [-t socket_timeout]
382N/A
135N/A --readonly Read-only operation; modifying operations disallowed
157N/A --rebuild Re-build the catalog from pkgs in depot
445N/A Cannot be used with --readonly
451N/A --proxy-base The url to use as the base for generating internal
451N/A redirects and content.
26N/A"""
135N/A sys.exit(2)
14N/A
382N/Aclass OptionError(Exception):
429N/A """Option exception. """
14N/A
404N/A def __init__(self, *args):
404N/A Exception.__init__(self, *args)
30N/A
382N/Aif __name__ == "__main__":
30N/A
382N/A port = PORT_DEFAULT
382N/A threads = THREADS_DEFAULT
382N/A socket_timeout = SOCKET_TIMEOUT_DEFAULT
382N/A readonly = READONLY_DEFAULT
382N/A rebuild = REBUILD_DEFAULT
429N/A reindex = REINDEX_DEFAULT
451N/A proxy_base = None
258N/A
382N/A if "PKG_REPO" in os.environ:
382N/A repo_path = os.environ["PKG_REPO"]
382N/A else:
382N/A repo_path = REPO_PATH_DEFAULT
30N/A
54N/A try:
382N/A parsed = set()
382N/A opts, pargs = getopt.getopt(sys.argv[1:], "d:np:s:t:",
451N/A ["readonly", "rebuild", "proxy-base=", "refresh-index"])
451N/A opt = None
135N/A for opt, arg in opts:
382N/A if opt in parsed:
382N/A raise OptionError, "Each option may only be " \
382N/A "specified once."
382N/A else:
382N/A parsed.add(opt)
382N/A
135N/A if opt == "-n":
135N/A sys.exit(0)
135N/A elif opt == "-d":
382N/A repo_path = arg
135N/A elif opt == "-p":
135N/A port = int(arg)
382N/A elif opt == "-s":
382N/A threads = int(arg)
382N/A if threads < THREADS_MIN:
382N/A raise OptionError, \
382N/A "minimum value is %d" % THREADS_MIN
382N/A if threads > THREADS_MAX:
382N/A raise OptionError, \
382N/A "maximum value is %d" % THREADS_MAX
382N/A elif opt == "-t":
382N/A socket_timeout = int(arg)
135N/A elif opt == "--readonly":
382N/A readonly = True
157N/A elif opt == "--rebuild":
382N/A rebuild = True
429N/A elif opt == "--refresh-index":
429N/A # Note: This argument is for internal use
429N/A # only. It's used when pkg.depotd is reexecing
429N/A # itself and needs to know that's the case.
429N/A # This flag is purposefully omitted in usage.
429N/A # The supported way to forcefully reindex is to
429N/A # kill any pkg.depot using that directory,
429N/A # remove the index directory, and restart the
429N/A # pkg.depot process. The index will be rebuilt
429N/A # automatically on startup.
429N/A reindex = True
451N/A elif opt == "--proxy-base":
451N/A # Attempt to decompose the url provided into
451N/A # its base parts. This is done so we can
451N/A # remove any scheme information since we
451N/A # don't need it.
451N/A scheme, netloc, path, params, query, \
451N/A fragment = urlparse.urlparse(arg,
451N/A allow_fragments=0)
451N/A
451N/A # Rebuild the url without the scheme and
451N/A # remove the leading // urlunparse adds.
451N/A proxy_base = urlparse.urlunparse(("", netloc,
451N/A path, params, query, fragment)
451N/A ).lstrip("//")
451N/A
135N/A except getopt.GetoptError, e:
404N/A print "pkg.depotd: %s" % e.msg
382N/A usage()
382N/A except OptionError, e:
382N/A print "pkg.depotd: option: %s -- %s" % (opt, e)
382N/A usage()
382N/A except (ArithmeticError, ValueError):
382N/A print "pkg.depotd: illegal option value: %s specified " \
382N/A "for option: %s" % (arg, opt)
135N/A usage()
451N/A
445N/A if rebuild and reindex:
445N/A print "--refresh-index cannot be used with --rebuild"
445N/A usage()
445N/A if rebuild and readonly:
445N/A print "--readonly cannot be used with --rebuild"
445N/A usage()
445N/A if reindex and readonly:
445N/A print "--readonly cannot be used with --refresh-index"
445N/A usage()
451N/A
429N/A # If the program is going to reindex, the port is irrelevant since
429N/A # the program will not bind to a port.
429N/A if not reindex:
429N/A available, msg = port_available(None, port)
429N/A if not available:
429N/A print "pkg.depotd: unable to bind to the specified " \
429N/A "port: %d. Reason: %s" % (port, msg)
429N/A sys.exit(1)
386N/A
382N/A try:
382N/A face.set_content_root(os.environ["PKG_DEPOT_CONTENT"])
382N/A except KeyError:
382N/A pass
382N/A
382N/A scfg = config.SvrConfig(repo_path, AUTH_DEFAULT)
382N/A
382N/A if rebuild:
382N/A scfg.destroy_catalog()
382N/A
382N/A if readonly:
382N/A scfg.set_read_only()
382N/A
382N/A try:
382N/A scfg.init_dirs()
382N/A except EnvironmentError, e:
382N/A print "pkg.depotd: an error occurred while trying to " \
382N/A "initialize the depot repository directory " \
382N/A "structures:\n%s" % e
382N/A sys.exit(1)
382N/A
429N/A if reindex:
451N/A scfg.acquire_catalog(rebuild=False)
429N/A scfg.catalog.run_update_index()
429N/A sys.exit(0)
429N/A
26N/A scfg.acquire_in_flight()
26N/A scfg.acquire_catalog()
26N/A
428N/A try:
428N/A root = cherrypy.Application(repo.Repository(scfg))
428N/A except rc.InvalidAttributeValueError, e:
428N/A emsg("pkg.depotd: repository.conf error: %s" % e)
428N/A sys.exit(1)
428N/A
382N/A # We have to override cherrypy's default response_class so that we
382N/A # have access to the write() callable to stream data directly to the
382N/A # client.
382N/A root.wsgiapp.response_class = depot.DepotResponse
382N/A
451N/A # Setup our basic server configuration.
382N/A cherrypy.config.update({
382N/A "environment": "production",
382N/A "checker.on": True,
382N/A "log.screen": True,
382N/A "server.socket_port": port,
382N/A "server.thread_pool": threads,
382N/A "server.socket_timeout": socket_timeout
382N/A })
382N/A
451N/A # Now build our site configuration.
382N/A conf = {
382N/A "/robots.txt": {
382N/A "tools.staticfile.on": True,
382N/A "tools.staticfile.filename": os.path.join(face.content_root,
382N/A "robots.txt")
382N/A },
382N/A "/static": {
382N/A "tools.staticdir.on": True,
382N/A "tools.staticdir.root": face.content_root,
382N/A "tools.staticdir.dir": ""
382N/A }
382N/A }
145N/A
451N/A if proxy_base:
451N/A # This changes the base URL for our server, and is primarily
451N/A # intended to allow our depot process to operate behind Apache
451N/A # or some other webserver process.
451N/A #
451N/A # Visit the following URL for more information:
451N/A # http://cherrypy.org/wiki/BuiltinTools#tools.proxy
451N/A proxy_conf = {
451N/A "tools.proxy.on": True,
451N/A "tools.proxy.local": "",
451N/A "tools.proxy.base": proxy_base
451N/A }
451N/A
451N/A if "/" not in conf:
451N/A conf["/"] = {}
451N/A
451N/A # Now merge or add our proxy configuration information into the
451N/A # existing configuration.
451N/A for entry in proxy_conf:
451N/A conf["/"][entry] = proxy_conf[entry]
451N/A
217N/A try:
382N/A cherrypy.quickstart(root, config = conf)
382N/A except:
382N/A print "pkg.depotd: unknown error starting depot, illegal " \
382N/A "option value specified?"
382N/A usage()
217N/A