pkg/src/depot.py

	depot.py revision 589
1516N/A#!/usr/bin/python2.4
290N/A#
290N/A# CDDL HEADER START
290N/A#
290N/A# The contents of this file are subject to the terms of the
290N/A# Common Development and Distribution License (the "License").
290N/A# You may not use this file except in compliance with the License.
290N/A#
290N/A# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
290N/A# or http://www.opensolaris.org/os/licensing.
290N/A# See the License for the specific language governing permissions
290N/A# and limitations under the License.
290N/A#
290N/A# When distributing Covered Code, include this CDDL HEADER in each
290N/A# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
290N/A# If applicable, add the following below this CDDL HEADER, with the
290N/A# fields enclosed by brackets "[]" replaced with your own identifying
290N/A# information: Portions Copyright [yyyy] [name of copyright owner]
290N/A#
290N/A# CDDL HEADER END
290N/A#
849N/A# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
395N/A# Use is subject to license terms.
395N/A#
290N/A
883N/A# pkg.depotd - package repository daemon
454N/A
290N/A# XXX The prototype pkg.depotd combines both the version management server that
448N/A# answers to pkgsend(1) sessions and the HTTP file server that answers to the
290N/A# various GET operations that a pkg(1) client makes.  This split is expected to
290N/A# be made more explicit, by constraining the pkg(1) operations such that they
290N/A# can be served as a typical HTTP/HTTPS session.  Thus, pkg.depotd will reduce
383N/A# to a special purpose HTTP/HTTPS server explicitly for the version management
290N/A# operations, and must manipulate the various state files--catalogs, in
395N/A# particular--such that the pkg(1) pull client can operately accurately with
290N/A# only a basic HTTP/HTTPS server in place.
395N/A
849N/A# XXX We should support simple "last-modified" operations via HEAD queries.
1516N/A
290N/A# XXX Although we pushed the evaluation of next-version, etc. to the pull
849N/A# client, we should probably provide a query API to do same on the server, for
290N/A# dumb clients (like a notification service).
290N/A
290N/A# The default authority for the depot.
290N/AAUTH_DEFAULT = "opensolaris.org"
383N/A# The default repository path.
290N/AREPO_PATH_DEFAULT = "/var/pkg/repo"
290N/A# The default path for static and other web content.
290N/ACONTENT_PATH_DEFAULT = "/usr/share/lib/pkg"
290N/A# The default port to serve data from.
290N/APORT_DEFAULT = 80
290N/A# The minimum number of threads allowed.
290N/ATHREADS_MIN = 1
290N/A# The default number of threads to start.
465N/ATHREADS_DEFAULT = 10
465N/A# The maximum number of threads that can be started.
465N/ATHREADS_MAX = 100
1516N/A# The default server socket timeout in seconds. We want this to be longer than
465N/A# the normal default of 10 seconds to accommodate clients with poor quality
465N/A# connections.
465N/ASOCKET_TIMEOUT_DEFAULT = 60
1516N/A# Whether modify operations should be allowed.
465N/AREADONLY_DEFAULT = False
465N/A# Whether the repository catalog should be rebuilt on startup.
465N/AREBUILD_DEFAULT = False
465N/A# Whether the indexes should be rebuilt
465N/AREINDEX_DEFAULT = False
465N/A# Not in mirror mode by default
465N/AMIRROR_DEFAULT = False
1099N/A
465N/Aimport getopt
1513N/Aimport logging
1513N/Aimport os
1514N/Aimport os.path
1513N/Aimport sys
1513N/Aimport urlparse
1513N/A
1099N/Atry:
1513N/A        import cherrypy
708N/A        version = cherrypy.__version__.split('.')
1391N/A        if map(int, version) < [3, 1, 0]:
1391N/A                raise ImportError
1391N/A        elif map(int, version) >= [3, 2, 0]:
1391N/A                raise ImportError
1391N/Aexcept ImportError:
1391N/A        print """cherrypy 3.1.0 or greater (but less than 3.2.0) is """ \
1391N/A            """required to use this program."""
1391N/A        sys.exit(2)
1391N/A
1391N/Aimport pkg.server.face as face
1391N/Aimport pkg.server.config as config
742N/Aimport pkg.server.depot as depot
742N/Aimport pkg.server.repository as repo
742N/Aimport pkg.server.repositoryconfig as rc
742N/Afrom pkg.misc import port_available, msg, emsg
742N/A
742N/Aclass LogSink(object):
1099N/A        """This is a dummy object that we can use to discard log entries
742N/A        without relying on non-portable interfaces such as /dev/null."""
941N/A
941N/A        def write(self, *args, **kwargs):
941N/A                """Discard the bits."""
941N/A                pass
941N/A
941N/A        def flush(self, *args, **kwargs):
1099N/A                """Discard the bits."""
941N/A                pass
1191N/A
1513N/Adef usage(text):
1191N/A        if text:
1191N/A                emsg(text)
1191N/A
1191N/A        print """\
1191N/AUsage: /usr/lib/pkg.depotd [-d repo_dir] [-p port] [-s threads]
1191N/A           [-t socket_timeout] [--content-root] [--log-access dest]
290N/A           [--log-errors dest] [--mirror] [--proxy-base url] [--readonly]
290N/A           [--rebuild]
290N/A
395N/A        --content-root  The file system path to the directory containing the
395N/A                        the static and other web content used by the depot's
290N/A                        browser user interface.  The default value is
395N/A                        '/usr/share/lib/pkg'.
395N/A        --log-access    The destination for any access related information
290N/A                        logged by the depot process.  Possible values are:
395N/A                        stderr, stdout, none, or an absolute pathname.  The
395N/A                        default value is stdout if stdout is a tty; otherwise
290N/A                        the default value is none.
395N/A        --log-errors    The destination for any errors or other information
395N/A                        logged by the depot process.  Possible values are:
1302N/A                        stderr, stdout, none, or an absolute pathname.  The
1302N/A                        default value is stderr.
1302N/A        --mirror        Package mirror mode; publishing and metadata operations
290N/A                        disallowed.  Cannot be used with --readonly or
448N/A                        --rebuild.
448N/A        --proxy-base    The url to use as the base for generating internal
534N/A                        redirects and content.
534N/A        --readonly      Read-only operation; modifying operations disallowed.
534N/A                        Cannot be used with --mirror or --rebuild.
534N/A        --rebuild       Re-build the catalog from pkgs in depot.  Cannot be
534N/A                        used with --mirror or --readonly.
534N/A"""
534N/A        sys.exit(2)
290N/A
290N/Aclass OptionError(Exception):
954N/A        """Option exception. """
954N/A
954N/A        def __init__(self, *args):
954N/A                Exception.__init__(self, *args)
534N/A
1099N/Aif __name__ == "__main__":
290N/A
1191N/A        port = PORT_DEFAULT
1191N/A        threads = THREADS_DEFAULT
1191N/A        socket_timeout = SOCKET_TIMEOUT_DEFAULT
1516N/A        readonly = READONLY_DEFAULT
290N/A        rebuild = REBUILD_DEFAULT
290N/A        reindex = REINDEX_DEFAULT
290N/A        proxy_base = None
661N/A        mirror = MIRROR_DEFAULT
290N/A
290N/A        if "PKG_REPO" in os.environ:
290N/A                repo_path = os.environ["PKG_REPO"]
395N/A        else:
290N/A                repo_path = REPO_PATH_DEFAULT
290N/A
290N/A        try:
1483N/A                content_root = os.environ["PKG_DEPOT_CONTENT"]
290N/A        except KeyError:
1498N/A                try:
1498N/A                        content_root = os.path.join(os.environ['PKG_HOME'],
290N/A                            'share/lib/pkg')
395N/A                except KeyError:
430N/A                        content_root = CONTENT_PATH_DEFAULT
395N/A
1544N/A        # By default, if the destination for a particular log type is not
1557N/A        # specified, this is where we will send the output.
1506N/A        log_routes = {
395N/A            "access": "none",
395N/A            "errors": "stderr"
424N/A        }
1024N/A        log_opts = ["--log-%s" % log_type for log_type in log_routes]
395N/A
395N/A        # If stdout is a tty, then send access output there by default instead
395N/A        # of discarding it.
578N/A        if os.isatty(sys.stdout.fileno()):
1228N/A                log_routes["access"] = "stdout"
1172N/A
395N/A        opt = None
661N/A        try:
1099N/A                long_opts = ["content-root=", "mirror", "proxy-base=",
661N/A                    "readonly", "rebuild", "refresh-index"]
395N/A                for opt in log_opts:
849N/A                        long_opts.append("%s=" % opt.lstrip('--'))
290N/A                opts, pargs = getopt.getopt(sys.argv[1:], "d:np:s:t:",
395N/A                    long_opts)
395N/A                for opt, arg in opts:
395N/A                        if opt == "-n":
395N/A                                sys.exit(0)
395N/A                        elif opt == "-d":
395N/A                                repo_path = arg
395N/A                        elif opt == "-p":
395N/A                                port = int(arg)
395N/A                        elif opt == "-s":
395N/A                                threads = int(arg)
395N/A                                if threads < THREADS_MIN:
395N/A                                        raise OptionError, \
395N/A                                            "minimum value is %d" % THREADS_MIN
290N/A                                if threads > THREADS_MAX:
290N/A                                        raise OptionError, \
395N/A                                            "maximum value is %d" % THREADS_MAX
395N/A                        elif opt == "-t":
1231N/A                                socket_timeout = int(arg)
1557N/A                        elif opt == "--content-root":
1557N/A                                if arg == "":
395N/A                                        raise OptionError, "You must specify " \
395N/A                                            "a directory path."
395N/A                                content_root = arg
395N/A                        elif opt in log_opts:
395N/A                                if arg is None or arg == "":
395N/A                                        raise OptionError, \
395N/A                                            "You must specify a log " \
395N/A                                            "destination."
395N/A                                log_routes[opt.lstrip("--log-")] = arg
395N/A                        elif opt == "--mirror":
395N/A                                mirror = True
290N/A                        elif opt == "--proxy-base":
290N/A                                # Attempt to decompose the url provided into
430N/A                                # its base parts.  This is done so we can
395N/A                                # remove any scheme information since we
395N/A                                # don't need it.
395N/A                                scheme, netloc, path, params, query, \
395N/A                                    fragment = urlparse.urlparse(arg,
1302N/A                                    allow_fragments=0)
395N/A
395N/A                                # Rebuild the url without the scheme and
290N/A                                # remove the leading // urlunparse adds.
395N/A                                proxy_base = urlparse.urlunparse(("", netloc,
1024N/A                                    path, params, query, fragment)
413N/A                                    ).lstrip("//")
1544N/A                        elif opt == "--readonly":
1557N/A                                readonly = True
1506N/A                        elif opt == "--rebuild":
413N/A                                rebuild = True
413N/A                        elif opt == "--refresh-index":
1024N/A                                # Note: This argument is for internal use
395N/A                                # only. It's used when pkg.depotd is reexecing
395N/A                                # itself and needs to know that's the case.
413N/A                                # This flag is purposefully omitted in usage.
395N/A                                # The supported way to forcefully reindex is to
395N/A                                # kill any pkg.depot using that directory,
413N/A                                # remove the index directory, and restart the
395N/A                                # pkg.depot process. The index will be rebuilt
395N/A                                # automatically on startup.
395N/A                                reindex = True
395N/A        except getopt.GetoptError, e:
395N/A                usage("pkg.depotd: %s" % e.msg)
395N/A        except OptionError, e:
1191N/A                usage("pkg.depotd: option: %s -- %s" % (opt, e))
1452N/A        except (ArithmeticError, ValueError):
1231N/A                usage("pkg.depotd: illegal option value: %s specified " \
395N/A                    "for option: %s" % (arg, opt))
395N/A
424N/A        if rebuild and reindex:
395N/A                usage("--refresh-index cannot be used with --rebuild")
742N/A        if rebuild and (readonly or mirror):
742N/A                usage("--readonly and --mirror cannot be used with --rebuild")
742N/A        if reindex and (readonly or mirror):
742N/A                usage("--readonly and --mirror cannot be used with " \
742N/A                    "--refresh-index")
742N/A
742N/A        # If the program is going to reindex, the port is irrelevant since
742N/A        # the program will not bind to a port.
742N/A        if not reindex:
742N/A                available, msg = port_available(None, port)
742N/A                if not available:
395N/A                        print "pkg.depotd: unable to bind to the specified " \
395N/A                            "port: %d. Reason: %s" % (port, msg)
395N/A                        sys.exit(1)
395N/A
395N/A        scfg = config.SvrConfig(repo_path, content_root, AUTH_DEFAULT)
954N/A
954N/A        if rebuild:
954N/A                scfg.destroy_catalog()
954N/A
954N/A        if readonly:
954N/A                scfg.set_read_only()
954N/A
395N/A        if mirror:
1483N/A                scfg.set_mirror()
1483N/A
1483N/A        try:
1483N/A                scfg.init_dirs()
395N/A        except (RuntimeError, EnvironmentError), e:
1099N/A                print "pkg.depotd: an error occurred while trying to " \
1099N/A                    "initialize the depot repository directory " \
395N/A                    "structures:\n%s" % e
1498N/A                sys.exit(1)
1498N/A
691N/A        # Setup our global configuration.
691N/A        # Global cherrypy configuration
691N/A        gconf = {
395N/A            "environment": "production",
395N/A            "checker.on": True,
395N/A            "log.screen": False,
395N/A            "server.socket_host": "0.0.0.0",
395N/A            "server.socket_port": port,
290N/A            "server.thread_pool": threads,
395N/A            "server.socket_timeout": socket_timeout,
395N/A            "tools.log_headers.on": True
591N/A        }
591N/A
591N/A        log_type_map = {
1505N/A            "errors": {
1505N/A                "param": "log.error_file",
1505N/A                "attr": "error_log"
1505N/A            },
1632N/A            "access": {
1632N/A                "param": "log.access_file",
1632N/A                "attr": "access_log"
1632N/A            }
395N/A        }
395N/A
290N/A        for log_type in log_type_map:
290N/A                dest = log_routes[log_type]
290N/A                if dest in ("stdout", "stderr", "none"):
290N/A                        if dest == "none":
290N/A                                h = logging.StreamHandler(LogSink())
290N/A                        else:
290N/A                                h = logging.StreamHandler(eval("sys.%s" % \
290N/A                                    dest))
290N/A
290N/A                        h.setLevel(logging.DEBUG)
290N/A                        h.setFormatter(cherrypy._cplogging.logfmt)
290N/A                        log_obj = eval("cherrypy.log.%s" % \
290N/A                            log_type_map[log_type]["attr"])
290N/A                        log_obj.addHandler(h)
395N/A                        # Since we've replaced cherrypy's log handler with our
395N/A                        # own, we don't want the output directed to a file.
290N/A                        dest = ""
290N/A
290N/A                gconf[log_type_map[log_type]["param"]] = dest
290N/A
290N/A        # Now build our site configuration.
395N/A        conf = {
395N/A            "/": {
395N/A                # We have to override cherrypy's default response_class so that
290N/A                # we have access to the write() callable to stream data
395N/A                # directly to the client.
395N/A                "wsgi.response_class": depot.DepotResponse,
395N/A            },
395N/A            "/robots.txt": {
591N/A                "tools.staticfile.on": True,
591N/A                "tools.staticfile.filename": os.path.join(scfg.web_static_root,
591N/A                    "robots.txt")
591N/A            },
691N/A            "/static": {
691N/A                "tools.staticdir.on": True,
691N/A                "tools.staticdir.root": scfg.web_static_root,
691N/A                "tools.staticdir.dir": ""
290N/A            }
290N/A        }
290N/A
290N/A        if proxy_base:
290N/A                # This changes the base URL for our server, and is primarily
591N/A                # intended to allow our depot process to operate behind Apache
591N/A                # or some other webserver process.
691N/A                #
691N/A                # Visit the following URL for more information:
290N/A                #    http://cherrypy.org/wiki/BuiltinTools#tools.proxy
395N/A                proxy_conf = {
395N/A                        "tools.proxy.on": True,
290N/A                        "tools.proxy.local": "",
395N/A                        "tools.proxy.base": proxy_base
395N/A                }
395N/A
395N/A                if "/" not in conf:
290N/A                        conf["/"] = {}
290N/A
290N/A                # Now merge or add our proxy configuration information into the
395N/A                # existing configuration.
395N/A                for entry in proxy_conf:
395N/A                        conf["/"][entry] = proxy_conf[entry]
395N/A
395N/A        cherrypy.config.update(gconf)
395N/A
290N/A        # Now that our logging, etc. has been setup, it's safe to perform any
290N/A        # remaining preparation.
290N/A        if reindex:
290N/A                scfg.acquire_catalog(rebuild=False)
290N/A                scfg.catalog.run_update_index()
430N/A                sys.exit(0)
290N/A
290N/A        scfg.acquire_in_flight()
395N/A        scfg.acquire_catalog()
290N/A
395N/A        try:
506N/A                root = cherrypy.Application(repo.Repository(scfg))
506N/A        except rc.InvalidAttributeValueError, e:
506N/A                emsg("pkg.depotd: repository.conf error: %s" % e)
506N/A                sys.exit(1)
506N/A
506N/A        try:
506N/A                cherrypy.quickstart(root, config=conf)
506N/A        except:
834N/A                usage("pkg.depotd: unknown error starting depot, illegal " \
506N/A                    "option value specified?")
506N/A
506N/A