#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
#
import atexit
import errno
import getopt
import gettext
import locale
import logging
import os
import shutil
import simplejson
import six
import socket
import stat
import sys
import traceback
import warnings
import pkg
orig_cwd = None
# exit codes
#
# This is a simple python script, run from the method script that starts
# svc:/application/pkg/system-repository:default.
#
# It writes an Apache configuration that is used to serve responses to pkg
# services to those clients, accessing external repositories.
# file:// repositories on the system running the system repository are also
# exposed to pkg clients, via Alias directives.
#
# Apache configuration.
#
# The following filesystem locations are used:
#
# variable default install path description
# --------- --------------------- ------------
#
# all of the above can be modified with command line arguments.
#
# static string with our versions response
SYSREPO_VERSIONS_STR = """\
pkg-server {0}
publisher 0
versions 0
catalog 1
file 1
syspub 0
manifest 0
pass
def cleanup():
"""To be called at program finish."""
pass
"""Emit an error message prefixed by the command name """
if cmd:
pkg_cmd = "pkg.sysrepo "
else:
pkg_cmd = "pkg.sysrepo: "
# If we get passed something like an Exception, we can convert
# it down to a string.
# If the message starts with whitespace, assume that it should come
# *before* the command-name prefix.
# This has to be a constant value as we can't reliably get our actual
# program name on all platforms.
"""Emit a usage message and optionally prefix it with a more
specific error message. Causes program to exit.
"""
if usage_error:
msg(_("""\
Usage:
pkg.sysrepo -p <port> [-R image_root] [ -c cache_dir] [-h hostname]
[-l logs_dir] [-r runtime_dir] [-s cache_size] [-t template_dir]
[-T http_timeout] [-w http_proxy] [-W https_proxy]
"""))
"""Return a pkg.client.api.ImageInterface for the provided
image directory."""
if not image_dir:
image_dir = "/"
api_inst = None
try:
tracker, None, PKG_CLIENT_NAME)
raise SysrepoException(
_("Unable to get image at {dir}: {reason}").format(
# restore the current directory, which ImageInterace had changed
return api_inst
""" Follow HTTP redirects from servers. Needed so that we can create
RewriteRules for all repository URLs that pkg clients may encounter.
We return a sorted list of URIs that were found having followed all
redirects in 'uri_list'. We also return a boolean, True if we timed out
when following any of the URIs.
"""
""" A HTTPRedirectHandler that saves URIs we've been
redirected to along the path to our eventual destination."""
return HTTPRedirectHandler.redirect_request(
continue
# otherwise, open a known url to check for redirects
try:
# We need to log this, and carry on - the url
# could become available at a later date.
msg(_("WARNING: unable to access {uri} when checking "
"""Determine if pub_info and no_uri_pubs objects, which may have been
decoded from a json representation are valid, raising a SysrepoException
if they are not.
We use the api_inst to sanity-check that all publishers configured in
the image are represented in pub_info or no_uri_pubs, and that their
URIs are present.
SysrepoExceptions are raised with developer-oriented debug messages
which are not to be translated or shown to users.
"""
# validate the structure of the pub_info object
uri))
uri_info))
raise SysrepoException("{0} does not have 6 "
# props [0] and [3] must be strings
raise SysrepoException("indices 0 and 3 of {0} "
# prop[5] must be a string, either "file" or "dir"
# and prop[0] must start with file://
raise SysrepoException("index 5 of {0} is not a "
"basestring or is not 'file' or 'dir'".format(
props))
# validate the structure of the no_uri_pubs object
for item in no_uri_pubs:
raise SysrepoException(
# check that we have entries for each URI for each publisher.
# (we may have more URIs than these, due to server-side http redirects
# that are not reflected as origins or mirrors in the image itself)
continue
return
"""Loads information about the publishers configured for the
given ImageInterface from image_dir in a format identical to that
returned by _get_publisher_info(..) that is, a dictionary mapping
URIs to a list of lists. An example entry might be:
pub_info[uri] = [[prefix, cert, key, hash of the uri, proxy], ... ]
and a list of publishers which have no origin or mirror URIs.
If the cache doesn't exist, or is in a format we don't recognise, or
we've managed to determine that it's stale, we return None, None
indicating that the publisher_info must be rebuilt.
"""
pub_info = None
no_uri_pubs = None
try:
try:
except OSError as e:
return None, None
else:
raise
# the cache must be a regular file
raise IOError("not a regular file")
try:
except simplejson.JSONDecodeError:
error(_("Invalid config cache file at {0} "
"generating fresh configuration.").format(
return None, None
error(_("Invalid config cache at {0} "
"generating fresh configuration.").format(
return None, None
# sanity-check the cached configuration
try:
except SysrepoException as e:
error(_("Invalid config cache at {0} "
"generating fresh configuration.").format(
return None, None
# If we have any problems loading the publisher info, we explain why.
except IOError as e:
**locals()))
return None, None
return pub_info, no_uri_pubs
"""Stores a given pair of (uri_pub_map, no_uri_pubs) objects to a
configuration cache file beneath image_dir."""
try:
try:
# if the cache exists, it must be a file
raise IOError("not a regular file")
except IOError as e:
# IOError has been merged into OSError in Python 3.4,
# so we need a special case here.
if str(e) == "not a regular file":
raise
except OSError:
pass
except IOError as e:
**locals()))
"""Checks the given proxy string to make sure that it does not contain
any authentication details since these are not supported by ProxyRemote.
"""
# If we don't have any authentication details, return.
return True
return False
"""Returns information about the publishers configured for the given
ImageInterface.
The first item returned is a map of uris to a list of lists of the form
[[prefix, cert, key, hash of the uri, proxy, uri type], ... ]
The second item returned is a list of publisher prefixes which specify
no uris.
Where possible, we attempt to load cached publisher information, but if
that cached information is stale or unavailable, we fall back to
querying the image for the publisher information, verifying repository
URIs and checking for redirects and write that information to the
cache."""
# the cache gets deleted by pkg.client.image.Image.save_config()
# any time publisher configuration changes are made.
if uri_pub_map:
return uri_pub_map, no_uri_pubs
# map URIs to (pub.prefix, cert, key, hash, proxy, utype) tuples
uri_pub_map = {}
no_uri_pubs = []
continue
# Only collect enabled URIs.
uris = [u
if not u.disabled
]
# Determine the proxies to use per URI
proxy_map = {}
# Apache can only use a single proxy, even
# if many are configured. Use the first we find.
# Apache's ProxyRemote directive does not allow proxies that
# require authentication.
raise SysrepoException("proxy value {val} "
"for {uri} is not supported.".format(
# We keep a field to store information about the type
# of URI we're looking at, which saves us
# from needing to make os.path.isdir(..) or
# os.path.isfile(..) calls when processing the template.
# This is important when we're rebuilding the
# configuration from cached publisher info and an
# file:// repository is temporarily unreachable.
utype = ""
# we only support p5p files and directory-based
# repositories of >= version 4.
utype = "dir"
raise SysrepoException(
_("file repository {0} does not "
raise SysrepoException(
_("file repository {0} cannot be "
"proxied. Only file "
"repositories of version 4 or "
utype = "file"
try:
except p5p.InvalidArchive:
raise SysrepoException(
_("unable to read p5p "
"archive file at {0}").format(
# so we just pull it from one of the RepositoryURIs.
)
if not uris:
# if we weren't able to follow all redirects, then we don't write a new
# cache, because it could be incomplete.
if not timed_out:
return uri_pub_map, no_uri_pubs
"""Sets ownership for cache directory as pkg5srv:bin"""
try:
raise SysrepoException(
_("Unable to chown to {user}:{group}: "
"{err}").format(
"""Writes the apache configuration for the system repository.
If http_proxy or http_proxy is supplied, it will override any proxy
values set in the image we're reading configuration from.
"""
try:
# check our hostname
# check our directories
raise SysrepoException(
try:
# set pkg5srv:bin as ownership for cache
# directory.
raise
# check our port
try:
raise SysrepoException(
except ValueError:
port))
# check our cache size
try:
if num <= 0:
raise SysrepoException(_("invalid cache size: "
except ValueError:
raise SysrepoException(
# check our proxy arguments - we can use a proxy to handle
# incoming http or https requests, but that proxy must use http.
("https_proxy", https_proxy)]:
if not val:
continue
try:
raise Exception(
_("scheme must be http"))
raise Exception("missing netloc")
if not _valid_proxy(val):
raise Exception("unsupported proxy")
except Exception as e:
raise SysrepoException(
_("invalid {key}: {val}: {err}").format(
# we're disabling unicode here because we want Mako to
# passthrough any filesystem path names, whatever the
# original encoding.
# Mako for Python 3 doesn't support disabling Unicode.
# our template expects cache size expressed in Kb
ipv6_addr="::1",
# socket.gethostbyname raise UnicodeDecodeError in Python 3
# for some input, such as '.'
raise SysrepoException(
_("Unable to write sysrepo_httpd.conf: {host}: "
raise SysrepoException(
"""Writes the crypto.txt file, containing keys and certificates
in order for the system repository to proxy to https repositories."""
try:
# Apache needs us to have some content in this file
if not written_crypto_content:
"# this space intentionally left blank\n")
raise SysrepoException(
"""Writes static html for all file-repository-based publishers that
is served as their publisher/0 responses. Responses for
non-file-based publishers are handled by rewrite rules in our
Apache configuration."""
try:
# build a version of our uri_pub_map, keyed by publisher
pub_uri_map = {}
for uri in uri_pub_map:
if pub not in pub_uri_map:
pub_uri_map[pub] = []
for pub in pub_uri_map:
publisher_text = \
SYSREPO_PUB_FILENAME]), "w")
raise SysrepoException(
"""Writes a static versions/0 response for the system repository."""
try:
"w")
raise SysrepoException(
"""Writes a static syspub/0 response for the system repository."""
try:
pub_prefixes = [
info[0]
]
pub_prefixes, 0)
raise SysrepoException(
"""Returns a string hash of the given URI"""
# Unicode-objects must be encoded before hashing
"""Change the ownership of all files under runtime_dir to our sysrepo
try:
raise SysrepoException(
_("Unable to chown to {user}:{group}: "
"{err}").format(
"""Destroys an old configuration."""
try:
raise SysrepoException(
"""Creates a new configuration for the system repository.
TODO: a way to map only given zones to given publishers
"""
try:
try:
except ValueError as err:
raise SysrepoException(
if http_timeout < 1:
raise SysrepoException(
_("http_timeout must a positive integer"))
try:
except SysrepoException as err:
raise SysrepoException(
_("unable to get publisher information: {0}").format(
err))
try:
raise SysrepoException(
except SysrepoException as err:
return ret
def main_func():
global orig_cwd
try:
except OSError as e:
try:
orig_cwd = None
except KeyError:
orig_cwd = None
# some sensible defaults
host = "127.0.0.1"
port = None
# an empty image_root means we don't get '//' in the below
# _get_image() deals with "" in a sane manner.
image_root = ""
cache_size = "1024"
http_timeout = 4
http_proxy = None
https_proxy = None
try:
"c:h:l:p:r:R:s:t:T:w:W:?", ["help"])
if opt == "-c":
elif opt == "-h":
elif opt == "-l":
elif opt == "-p":
elif opt == "-r":
elif opt == "-R":
elif opt == "-s":
elif opt == "-t":
elif opt == "-T":
elif opt == "-w":
elif opt == "-W":
else:
usage()
except getopt.GetoptError as e:
if not port:
usage(_("required port option missing."))
return ret
#
# Establish a specific exit status which means: "python barfed an exception"
# so that we can more easily detect these in testing of the CLI commands.
#
"""Catch exceptions raised by the main program function and then print
"""
try:
# Out of memory errors can be raised as EnvironmentErrors with
# an errno of ENOMEM, so in order to handle those exceptions
# with other errnos, we nest this try block and have the outer
# one handle the other instances.
try:
raise
except SystemExit as __e:
raise __e
except (PipeError, KeyboardInterrupt):
# Don't display any messages here to prevent possible further
# broken pipe (EPIPE) errors.
error(_("The sysrepo command appears out of sync with the "
"version is {client} while the library\nAPI version is "
except:
__ret = 99
return __ret
if __name__ == "__main__":
# Make all warnings be errors.
# disable ResourceWarning: unclosed file
try:
except IOError:
# Ignore python's spurious pipe problems.
pass