#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# pkg.depotd - package repository daemon
# XXX The prototype pkg.depotd combines both the version management server that
# answers to pkgsend(1) sessions and the HTTP file server that answers to the
# various GET operations that a pkg(1) client makes. This split is expected to
# be made more explicit, by constraining the pkg(1) operations such that they
# operations, and must manipulate the various state files--catalogs, in
# particular--such that the pkg(1) pull client can operately accurately with
# XXX Although we pushed the evaluation of next-version, etc. to the pull
# client, we should probably provide a query API to do same on the server, for
# dumb clients (like a notification service).
# The default path for static and other web content.
# cherrypy has a max_request_body_size parameter that determines whether the
# server should abort requests with REQUEST_ENTITY_TOO_LARGE when the request
# body is larger than the specified size (in bytes). The maximum size supported
# by cherrypy is 2048 * 1024 * 1024 - 1 (just short of 2048MB), but the default
# here is purposefully conservative.
# The minimum number of threads allowed.
# The default number of threads to start.
# The maximum number of threads that can be started.
# The default server socket timeout in seconds. We want this to be longer than
# the normal default of 10 seconds to accommodate clients with poor quality
# connections.
import getopt
import gettext
import locale
import logging
import os
import shlex
import six
import string
import subprocess
import sys
import tempfile
else:
try:
import cherrypy
# comparison requires same type, therefore list conversion is needed
raise ImportError
except ImportError:
print("""cherrypy 3.1.0 or greater is required to use this program.""",
import pkg
# Starting in CherryPy 3.2, its default dispatcher converts all punctuation to
# underscore. Since publisher name can contain the hyphen symbol "-", in order
# to let the dispatcher to find the correct page handler, we need to skip
# converting the hyphen symbol.
else:
"""This is a dummy object that we can use to discard log entries
"""Discard the bits."""
pass
"""Discard the bits."""
pass
"""Optionally emit a usage message and then exit using the specified
exit code."""
if text:
if not full:
# The full usage message isn't desired.
emsg(_("Try `pkg.depotd --help or -?' for more "
"information."))
print("""\
Usage: /usr/lib/pkg.depotd [-a address] [-d inst_root] [-p port] [-s threads]
[-t socket_timeout] [--cfg] [--content-root]
[--disable-ops op[/1][,...]] [--debug feature_list]
[--image-root dir] [--log-access dest] [--log-errors dest]
[--mirror] [--nasty] [--nasty-sleep] [--proxy-base url]
[--readonly] [--ssl-cert-file] [--ssl-dialog] [--ssl-key-file]
[--sort-file-max-size size] [--writable-root dir]
-a address The IP address on which to listen for connections. The
default value is 0.0.0.0 (INADDR_ANY) which will listen
on all active interfaces. To listen on all active IPv6
interfaces, use '::'.
-d inst_root The file system path at which the server should find its
repository data. Required unless PKG_REPO has been set
in the environment.
-p port The port number on which the instance should listen for
incoming package requests. The default value is 80 if
ssl certificate and key information has not been
provided; otherwise, the default value is 443.
-s threads The number of threads that will be started to serve
requests. The default value is 10.
-t timeout The maximum number of seconds the server should wait for
a response from a client before closing a connection.
The default value is 60.
--cfg The pathname of the file to use when reading and writing
depot configuration data, or a fully qualified service
fault management resource identifier (FMRI) of the SMF
service or instance to read configuration data from.
--content-root The file system path to the directory containing the
the static and other web content used by the depot's
browser user interface. The default value is
--disable-ops A comma separated list of operations that the depot
should not configure. If, for example, you wanted
to omit loading search v1, 'search/1' should be
provided as an argument, or to disable all search
operations, simply 'search'.
--debug The name of a debug feature to enable; or a whitespace
or comma separated list of features to enable.
Possible values are: headers, hash=sha1+sha256,
hash=sha256, hash=sha1+sha512t_256, hash=sha512t_256
--image-root The path to the image whose file information will be
used as a cache for file data.
--log-access The destination for any access related information
logged by the depot process. Possible values are:
stderr, stdout, none, or an absolute pathname. The
default value is stdout if stdout is a tty; otherwise
the default value is none.
--log-errors The destination for any errors or other information
logged by the depot process. Possible values are:
stderr, stdout, none, or an absolute pathname. The
default value is stderr.
--mirror Package mirror mode; publishing and metadata operations
disallowed. Cannot be used with --readonly or
--rebuild.
--nasty Instruct the server to misbehave. At random intervals
it will time-out, send bad responses, hang up on
clients, and generally be hostile. The option
takes a value (1 to 100) for how nasty the server
should be.
--nasty-sleep In nasty mode (see --nasty), how many seconds to
randomly sleep when a random sleep occurs.
--proxy-base The url to use as the base for generating internal
redirects and content.
--readonly Read-only operation; modifying operations disallowed.
Cannot be used with --mirror or --rebuild.
--ssl-cert-file The absolute pathname to a PEM-encoded Certificate file.
This option must be used with --ssl-key-file. Usage of
this option will cause the depot to only respond to SSL
requests on the provided port.
--ssl-dialog Specifies what method should be used to obtain the
passphrase needed to decrypt the file specified by
--ssl-key-file. Supported values are: builtin,
default value is builtin. If smf is specified, an
SMF FMRI must be provided using the --cfg option.
--ssl-key-file The absolute pathname to a PEM-encoded Private Key file.
This option must be used with --ssl-cert-file. Usage of
this option will cause the depot to only respond to SSL
requests on the provided port.
--sort-file-max-size
The maximum size of the indexer sort file. Used to
limit the amount of RAM the depot uses for indexing,
or increase it for speed.
--writable-root The path to a directory to which the program has write
access. Used with --readonly to allow server to
create needed files, such as search indices, without
needing write access to the package information.
Options:
--help or -?
Environment:
PKG_REPO Used as default inst_root if -d not provided.
PKG_DEPOT_CONTENT Used as default content_root if --content-root
not provided.""")
"""Option exception. """
if __name__ == "__main__":
# Track initial configuration values.
try:
except KeyError:
try:
except KeyError:
pass
opt = None
debug_features = []
disable_ops = []
repo_props = {}
user_cfg = None
try:
"content-root=", "debug=", "disable-ops=", "exit-ready",
"help", "image-root=", "log-access=", "log-errors=",
"llmirror", "mirror", "nasty=", "nasty-sleep=",
"proxy-base=", "readonly", "rebuild", "refresh-index",
"set-property=", "ssl-cert-file=", "ssl-dialog=",
"ssl-key-file=", "sort-file-max-size=", "writable-root="]
if opt == "-a":
elif opt == "-n":
elif opt == "-d":
elif opt == "-p":
elif opt == "-s":
if threads < THREADS_MIN:
raise OptionError(
"minimum value is {0:d}".format(
if threads > THREADS_MAX:
raise OptionError(
"maximum value is {0:d}".format(
elif opt == "-t":
elif opt == "--add-content":
elif opt == "--cfg":
elif opt == "--cfg-file":
elif opt == "--content-root":
elif opt == "--debug":
continue
# A list of features can be specified using a
# "," or any whitespace character as separators.
if "," in arg:
else:
# We also allow key=value debug flags, which
# get set in pkg.client.debugvalues
try:
except (AttributeError, ValueError):
pass
elif opt == "--disable-ops":
raise OptionError(
"An argument must be specified.")
for s in disableops:
if "/" in s:
else:
op = s
ver = "*"
if op not in \
raise OptionError(
"Invalid operation "
"'{0}'.".format(s))
elif opt == "--exit-ready":
elif opt == "--image-root":
elif opt == "--mirror":
elif opt == "--llmirror":
elif opt == "--nasty":
# ValueError is caught by caller.
raise OptionError("Invalid value "
"for nasty option.\n Please "
"choose a value between 1 and 100.")
elif opt == "--nasty-sleep":
# ValueError is caught by caller.
elif opt == "--proxy-base":
# Attempt to decompose the url provided into
# its base parts. This is done so we can
# remove any scheme information since we
# don't need it.
if not netloc:
raise OptionError("Unable to "
"determine the hostname from "
"the provided URL; please use a "
"fully qualified URL.")
raise OptionError("Invalid URL; http "
"and https are the only supported "
"schemes.")
# Rebuild the url with the sanitized components.
elif opt == "--readonly":
elif opt == "--rebuild":
elif opt == "--refresh-index":
# Note: This argument is for internal use
# only.
#
# This flag is purposefully omitted in usage.
# The supported way to forcefully reindex is to
# kill any pkg.depot using that directory,
# remove the index directory, and restart the
# pkg.depot process. The index will be rebuilt
# automatically on startup.
elif opt == "--set-property":
try:
except ValueError:
usage(_("property arguments must be of "
"the form '<section.property>="
"<value>'."))
elif opt == "--ssl-cert-file":
# Assume this is an override to clear
# the value.
arg = ""
raise OptionError("The path to "
"the Certificate file must be "
"absolute.")
raise OptionError("The specified "
"file does not exist.")
raise OptionError("The specified "
"pathname is not a file.")
elif opt == "--ssl-key-file":
# Assume this is an override to clear
# the value.
arg = ""
raise OptionError("The path to "
"the Private Key file must be "
"absolute.")
raise OptionError("The specified "
"file does not exist.")
raise OptionError("The specified "
"pathname is not a file.")
elif opt == "--ssl-dialog":
if arg != "builtin" and \
arg != "smf" and not \
raise OptionError("Invalid value "
"specified. Expected: builtin, "
"an SMF FMRI.")
if os_util.get_canonical_os_type() != \
"unix":
# Don't allow a somewhat
# insecure authentication method
# on some platforms.
raise OptionError("exec is "
"not a supported dialog "
"type for this operating "
"system.")
"exec:")[1])
raise OptionError("Invalid "
"file path specified for "
"exec.")
elif opt == "--sort-file-max-size":
elif opt == "--writable-root":
# Set accumulated values.
if debug_features:
if disable_ops:
if addresses:
if DebugValues:
# Build configuration object.
except OptionError as _e:
except (ArithmeticError, ValueError):
usage("pkg.depotd: illegal option value: {0} specified " \
if show_usage:
# If stdout is a tty, then send access output there by default instead
# of discarding it.
else:
# Check for invalid option combinations.
if rebuild and add_content:
usage("--add-content cannot be used with --rebuild")
usage("--refresh-index cannot be used with --rebuild")
usage("--readonly and --mirror cannot be used with --rebuild "
"or --add-content")
usage("--mirror cannot be used with --refresh-index")
usage("--readonly can only be used with --refresh-index if "
"--writable-root is used")
if image_root and not ll_mirror:
usage("--image-root can only be used with --llmirror.")
if image_root and writable_root:
usage("--image_root and --writable-root cannot be used "
"together.")
if image_root and inst_root:
usage("--image-root and -d cannot be used together.")
# If the image format changes this may need to be reexamined.
if image_root:
# Set any values using defaults if they weren't provided.
# Only use the first value for now; multiple bind addresses may be
# supported later.
if address:
elif not address:
if not inst_root:
usage("Either PKG_REPO or -d must be provided")
if not content_root:
usage("The --ssl-cert-file and --ssl-key-file options must "
"must both be provided when using either option.")
elif not port:
if ssl_cert_file and ssl_key_file:
else:
if not socket_timeout:
if not threads:
# If the program is going to reindex, the port is irrelevant since
# the program will not bind to a port.
if not exit_ready:
try:
except Exception as e:
emsg("pkg.depotd: unable to bind to the specified "
else:
# Not applicable if we're not going to serve content
# Any relative paths should be made absolute using pkg_root. 'pkg_root'
# is a special property that was added to enable internal deployment of
# multiple disparate versions of the pkg.depotd software.
if ssl_cert_file:
if ssl_cert_file == "none":
ssl_cert_file = None
if ssl_key_file:
if ssl_key_file == "none":
ssl_key_file = None
# Setup SSL if requested.
key_data = None
ssl_dialog != "builtin":
cmdline = None
p = None
try:
if cmdline:
else:
cmdargs = []
stderr=None)
p.wait()
emsg("pkg.depotd: an error occurred while "
"executing [{0}]; unable to obtain the "
"passphrase needed to decrypt the SSL "
__e))
if ssl_dialog == "smf":
# Assume the configuration target was an SMF
# FMRI and let svcprop fail with an error if
# it wasn't.
else:
# The key file requires decryption, but the user has requested
# exec-based authentication, so it will have to be decoded first
# to an un-named temporary file.
try:
except EnvironmentError as _e:
emsg("pkg.depotd: unable to read the SSL private key "
emsg("pkg.depotd: authentication or cryptography "
"failure while attempting to decode\nthe SSL "
else:
# Redirect the server to the decrypted key file.
# Setup our global configuration.
gconf = {
"checker.on": True,
"environment": "production",
"log.screen": False,
"server.max_request_body_size": MAX_REQUEST_BODY_SIZE,
"server.shutdown_timeout": 0,
"server.socket_host": address,
"server.socket_port": port,
"server.socket_timeout": socket_timeout,
"server.ssl_certificate": ssl_cert_file,
"server.ssl_private_key": ssl_key_file,
"server.thread_pool": threads,
"tools.log_headers.on": True,
"tools.encode.on": True,
"tools.encode.encoding": "utf-8",
}
# Despite its name, this only logs headers when there is an
# error; it's redundant with the debug feature enabled.
# Causes the headers of every request to be logged to the error
# log; even if an exception occurs.
"on_start_resource",
log_cfg = {
}
# If stdin is not a tty and the pkgdepot controller isn't being used,
# then assume process will be daemonized and redirect output.
# Ensure log handlers are setup to use the file descriptors for
# stdout and stderr as the Daemonizer (used for test suite and
# SMF service) requires this.
log_type_map = {
"errors": {
"param": "log.error_file",
"attr": "error_log"
},
"access": {
"param": "log.access_file",
"attr": "access_log"
}
}
for log_type in log_type_map:
if dest == "none":
else:
dest)))
# Since we've replaced cherrypy's log handler with our
# own, we don't want the output directed to a file.
dest = ""
elif dest:
# Now that our logging, etc. has been setup, it's safe to perform any
# remaining preparation.
# Initialize repository state.
if not readonly:
# Not readonly, so assume a new repository should be created.
try:
except sr.RepositoryExistsError:
# Already exists, nothing to do.
pass
try:
"sort_file_max_size")
# Automatically update search indexes on startup if not already
if reindex:
try:
# Only execute a index refresh here if --exit-ready was
# requested; it will be handled later in the setup
# process for other cases.
api_errors.ApiException) as e:
elif rebuild:
try:
except sr.RepositoryError as e:
except (search_errors.IndexingException,
api_errors.PermissionsException) as e:
elif add_content:
try:
except sr.RepositoryError as e:
except (search_errors.IndexingException,
api_errors.PermissionsException) as e:
# Ready to start depot; exit now if requested.
if exit_ready:
# Next, initialize depot.
if nasty:
else:
# Now build our site configuration.
conf = {
"/": {},
"/robots.txt": {
"tools.staticfile.on": True,
"robots.txt")
},
}
if proxy_base:
# This changes the base URL for our server, and is primarily
# intended to allow our depot process to operate behind Apache
# or some other webserver process.
#
# Visit the following URL for more information:
# http://cherrypy.org/wiki/BuiltinTools#tools.proxy
proxy_conf = {
"tools.proxy.on": True,
"tools.proxy.local": "",
"tools.proxy.base": proxy_base
}
# Now merge or add our proxy configuration information into the
# existing configuration.
for entry in proxy_conf:
if ll_mirror:
if reindex:
# Tell depot to update search indexes when possible;
# this is done as a background task so that packages
# can be served immediately while search indexes are
# still being updated.
# If stdin is not a tty and the pkgdepot controller isn't being used,
# then assume process should be daemonized.
# Translate the values in log_cfg into paths.
try:
emsg("pkg.depotd: unknown error starting depot server, " \
"illegal option value specified?")