# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Define the basic classes that all test cases are inherited from.
# The currently defined test case classes are:
#
# ApacheDepotTestCase
# CliTestCase
# ManyDepotTestCase
# Pkg5TestCase
# SingleDepotTestCase
# SingleDepotTestCaseCorruptImage
#
import baseline
import copy
import difflib
import errno
import gettext
import hashlib
import logging
import multiprocessing
import os
import pprint
import shutil
import signal
import six
import stat
import subprocess
import sys
import tempfile
import time
import unittest
import operator
import platform
import pty
import pwd
import re
import ssl
import textwrap
import threading
import traceback
else:
#
# These are initialized by pkg5testenv.setup_environment.
#
# Location of root of test suite.
# User's value for TEMPDIR
# Location of path of pkg bits.
#
# XXX?
#
"""An exception used to signal that all testing should cease.
This is a framework-internal exception that tests should not
raise"""
pass
"""An exception used to signal that a test was skipped.
Should be initialized with a string giving a more detailed
reason. Test cases can raise this to the framework
that some prerequisite of the test is unsatisfied. A string
explaining the error should be passed at construction. """
#
# Errors for which the traceback is likely not useful.
#
# Version test suite is known to work with.
topdivider = \
",---------------------------------------------------------------------\n"
botdivider = \
"`---------------------------------------------------------------------\n"
if comment is not None:
comm = ""
if line == "":
continue
return comm + "\n"
else:
return "<no comment>\n\n"
str = " Output Follows:\n"
if command is not None:
str += "| <no output>\n"
else:
return str
str = " Debug Buffer Follows:\n"
str += "| <no debug buffer>\n"
else:
return str
str = ""
else:
return str
"""This class is a special log handler to redirect logger output to
the test case class' debug() method.
"""
# Ensure logger messages output by unit tests are redirected
# to debug output so they are not shown by default.
# Needed for compatibility
#
# Some dns servers return results for unknown dns names to redirect
# callers to a common landing page. To avoid getting tripped up by
# these stupid servers make sure that bogus_url actually contains an
# syntactically invalid dns name so we'll never succeed at the lookup.
#
smf_cmds = { \
import sys
if __name__ == "__main__":
sys.exit(1)
"""}
self.__test_root = None
self.__base_port = None
self.coverage_env = {}
self.next_free_port = None
global g_proto_readable
if not g_proto_readable:
return self._testMethodName
return self.__suite_name
if self.__base_port is not None or \
self.next_free_port is not None:
raise RuntimeError("Setting the base port twice isn't "
"allowed")
"""Ensure proto area is readable by unprivileged user."""
try:
except:
raise TestStopException("proto area '{0} is not "
"readable by unprivileged user {1}".format(
"""Test that a regexp search matches text."""
return
raise self.failureException(
"""Perform the same logic as assertRaises, but then verify
that the stringified version of the exception contains the
regexp pattern.
Introduced in in python 2.7"""
try:
except excClass as e:
return
raise self.failureException(
"""Perform the same logic as assertRaises, but then verify that
the exception raised can be stringified."""
try:
except excClass as e:
str(e)
return
else:
#
# Uses property() to implements test_root as a read-only attribute.
#
if not self.__test_root:
return None
pass
"""Less featureful but inspired by subprocess.Popen.
Runs subprocess in a pty"""
#
# Note: In theory the right answer here is to subclass Popen,
# but we found that in practice we'd have to reimplement most
# of that class, because its handling of file descriptors is
# too brittle in its _execute_child() code.
#
# Use a list as a way to pass by reference
while True:
chunksz = 1024
# assume we hit EOF
break
# This is the arg handling protocol from Popen
else:
if shell:
if executable:
if executable is None:
if pid == 0:
try:
# Child
if env is None:
else:
except:
else:
outlist = []
t.start()
t.join()
"usepty not supported with stdin")
# If caller provides arguments as a string, the shell must
# process them.
wrapper = ""
if coverage:
if wrapper:
# Coverage command must be split into arguments.
while wrapper:
if su_wrap:
# This ensures that all parts of the command
# line to be passed to 'su -c' are passed as a
# single argument.
while su_wrap:
if prefix:
else:
# Space needed between su_wrap and wrapper.
if coverage:
if env_arg:
if not usepty:
if handle:
# Do nothing more.
return p
retcode = p.returncode
else:
# In Python 3, subprocess returns bytes, while our pkg CLI
# utilites' standard output and error streams return
# str (unicode). To mimic the behavior of CLI, we force the
# output to be str. This is a no-op in Python 2.
encoding = "utf-8"
# For testing encoding other than utf-8, we need to pass the
# encoding to fore_str.
# locale is a form of "x.y" and we ignore the C locale
if index > -1:
if out:
if stderr:
return retcode
s = str(s)
for x in s.splitlines():
if self.debug_output:
subsequent_indent="\t",
else:
if lines == []:
lines = [""]
else:
ins = ""
return self.__debug_buf
self.__debug_buf = s
"""If 'shell' is True, the wrapper will be returned as a tuple of
strings of the form (su_wrap, su_end). If 'shell' is False, the
wrapper willbe returned as a tuple of (args, ignore) where
'args' is a list of the commands and their arguments that should
come before the command being executed."""
if not su_wrap:
return "", ""
else:
su_user = ""
cov_env = [
"{0}={1}".format(*e)
]
su_wrap = ["su"]
if su_user:
if shell:
else:
# If this ever changes, cmdline_run must be updated.
if shell:
su_end = "'"
else:
su_end = ""
try:
except OSError as e:
raise e
"signing_certs", "produced")
"code_signing_certs")
"chain_certs")
#
# TMPDIR affects the behavior of mkdtemp and mkstemp.
# Setting this here should ensure that tests will make temp
# files and dirs inside the test directory rather than
# polluting /tmp.
#
# Create a pkglintrc file that points to our info.classification
# data, and doesn't exclude any shipped plugins.
{"info_classification_path":
DebugValues["smf_cmds_dir"] = \
# impl_tearDown exists so that we can ensure that this class's
# teardown is actually called. Sometimes, subclasses will
# implement teardown but forget to call the superclass teardown.
if self.__didteardown:
return
try:
except OSError:
# working directory of last resort.
#
# Kill depots before blowing away test dir-- otherwise
# the depot can race with the shutil.rmtree()
#
try:
except Exception as e:
#
# We have some sloppy subclasses which don't call the superclass
# setUp-- in which case the dir might not exist. Tolerate it.
#
# Also, avoid deleting our fakeroot since then we'd have to
# keep re-creating it.
#
if self.__test_root is not None and \
try:
except OSError as e:
else:
raise
# In reality this call does nothing.
if result is None:
try:
try:
except KeyboardInterrupt:
# Try hard to make sure we've done a teardown.
try:
except:
pass
raise TestStopException
except:
# teardown could fail too, esp. if setup failed...
try:
except:
pass
# Try hard to make sure we've done a teardown.
return
try:
except self.failureException:
except KeyboardInterrupt:
# Try hard to make sure we've done a teardown.
except TestSkippedException as err:
except:
try:
except KeyboardInterrupt:
except:
# Try hard to make sure we've done a teardown.
# Make sure we don't mark this error'd twice.
if not error_added:
if needtodie:
try:
except:
pass
raise TestStopException
if ok:
finally:
# make sure we restore our directory if it still exists.
try:
except OSError as e:
# If directory doesn't exist anymore it doesn't
# matter.
raise
#
# The following are utility functions for use by testcases.
#
"""Given a C program (as a string), compile it into the
executable given by outputfile. Outputfile should be
given as a relative path, and will be located below the
test prefix path. Additional compiler options should be
passed in 'opts'. Suitable for compiling small test
programs."""
#
# We use a series of likely compilers. At present we support
# this testing with SunStudio.
#
try:
except OSError as e:
raise
if prog_text:
else:
try:
# Make sure to use shell=True so that env.
# vars and $PATH are evaluated.
rc = s.returncode
except OSError: pass
except OSError: pass
raise RuntimeError(
"Compile failed: {0} --> {1:d}\n{2}".format(
if rc == 127:
continue
# so rc == 0
break
except OSError:
continue
try:
except OSError:
pass
if not found:
raise TestSkippedException(
"No suitable Sun Studio compiler found. "
"Tried: {0}. Try setting $CC to a valid"
else:
""" Make miscellaneous text files. Files can be a
single relative pathname, a list of relative pathnames,
or a hash mapping relative pathnames to specific contents.
If file contents are not specified, the pathname of the
file is placed into the file as default content. """
outpaths = []
#
# If files is a string, make it a list. Then, if it is
# a list, simply turn it into a dict where each file's
# contents is its own name, so that we get some uniqueness.
#
nfiles = {}
for f in files:
nfiles[f] = f
if prefix is None:
else:
# Ensure output paths are returned in consistent order.
assert not f.startswith("/"), \
("{0}: misc file paths must be relative!".format(f))
return outpaths
# Trim to ensure nice looking output.
# Place inside of test prefix.
if pfmri:
return t_path
"""Find the hash of pem representation the file."""
f.read(), default_backend())
"""Reduce runs of spaces down to a single space."""
"""Compare two JSON-encoded strings."""
try:
except AssertionError as e:
if msg:
msg += "\n"
msg=""):
"""Compare two strings."""
if msg:
msg += "\n"
"Actual output differed from expected output\n" + msg +
"""A helper function used to match child images with their
expected values so that they can be checked."""
raise RuntimeError("Got a different set of image names "
"than was expected.\nExpected:\n{0}\nSeen:\n{1}".format(
break
version=0):
"""Check that the parsable output in 'output' is what is
expected."""
try:
except Exception as e:
raise RuntimeError("JSON couldn't parse the "
"output.\nError was: {0}\nOutput was:\n{1}".format(
e, output))
else:
# It's difficult to check that space-available is correct in the
# test suite.
del outd["space-available"]
# While we could check for space-required, it just means lots of
# tests would need to be changed if we ever changed our size
# measurement and other tests should be ensuring that the number
# is correct.
del outd["space-required"]
# Do not check item-messages, since it will be checked
# somewhere else.
del outd["item-messages"]
# Add 4 to account for self, output, include, and outd.
"different set of keys for expected and outd. Those in "
"expected but not in outd:\n{0}\nThose in outd but not in "
"expected:\n{1}".format(
continue
ev = []
continue
"of {1} was expected to be\n{2} but was\n{3}".format(
if include:
# Assert all sections expicitly requested were matched.
suffix=""):
pairs in the provided section those from the 'config'
dictionary. The new config file is written to the supplied
test_root, returning the name of that new file.
Used to set keys to point to paths beneath our test_root,
which would otherwise be shipped as hard-coded paths, relative
to /.
"""
else:
return new_rcfile.name
"""Class for passing test results between processes."""
"timing"]
self.baseline_failures = []
"""Class for combining test results from different test cases."""
for l in self.list_attrs:
for l in self.list_attrs:
v += getattr(o, n)
baseline = None
self.mismatches = []
return _OverTheWireResults(self)
# Override the unittest version of this so that success is
# considered "matching the baseline"
""" Pull the ejection lever. Stop execution, doing as
much forcible cleanup as possible. """
try:
tdf()
except Exception as e:
pass
try:
except Exception as e:
pass
try:
except Exception as e:
pass
raise TestStopException()
mstr = "MATCH"
else:
mstr = "MISMATCH"
res = ""
for s in instr.splitlines():
return res
for s in instr.splitlines():
if s.strip() == "":
continue
assert self.archive_dir
if test.debug_output:
try:
except socketerror as e:
pass
else:
# If the test has failed without creating its directory,
# make it manually, so that we have a place to write out
# ERROR_INFO.
f.write("------------------DEBUG LOG---------------\n")
if info is not None:
f.write("\n\n------------------EXCEPTION---------------\n")
f.close()
# If we're debugging, we'll have had output since we
# announced the name of the test, so restate it.
if test.debug_output:
res = "match pass"
else:
"Successful Test", "# ")
else:
res = "."
else:
# Bail out completely if the 'bail on fail' flag is set
# but iff the result disagrees with the baseline.
# for a few special exceptions, we delete the traceback so
# as to elide it. use only when the traceback itself
# is not likely to be useful.
if errtype in ELIDABLE_ERRORS:
else:
# If we're debugging, we'll have had output since we
# announced the name of the test, so restate it.
if test.debug_output:
if errtype in ELIDABLE_ERRORS:
else:
res += "\n"
b = "match"
else:
b = "MISMATCH"
if errtype in ELIDABLE_ERRORS:
else:
"Error Information", "# ")
res = "e"
else:
res = "E"
else:
# Check to see if we should archive this baseline mismatch.
# Bail out completely if the 'bail on fail' flag is set
# but iff the result disagrees with the baseline.
res = ""
if dbgbuf != "":
if error is not None:
return res
# If we're debugging, we'll have had output since we
# announced the name of the test, so restate it.
if test.debug_output:
res += "\n"
else:
res = "match FAIL (expected: FAIL)"
"Failure Information", "# ")
res = "f"
else:
res = "F"
else:
# Check to see if we should archive this baseline mismatch.
# Bail out completely if the 'bail on fail' flag is set
# but iff the result disagrees with the baseline.
"""Python 2.7 adds formal support for skipped tests in unittest
For now, we'll record this as a success, but also save the
reason why we wanted to skip this test"""
res += "# As a result, all test cases in this class will " \
"result in errors!\n#\n"
if errtype in ELIDABLE_ERRORS:
else:
"Persistent Setup Error Information", "# ")
if errtype in ELIDABLE_ERRORS:
else:
"Persistent Teardown Error Information", "# ")
name += "|"
return
if test._testMethodDoc is not None:
del docs[-1]
for x in docs:
if not test.debug_output:
def find_names(s):
"""Find the module and class names for the given test suite."""
mod = l[0]
return mod, c
"""Construct a test result for use in the parallel test suite."""
return res
"""Function used to run the test suite in parallel.
The 'inq' parameter is the queue to pull from to get test suites.
The 'outq' parameter is the queue on which to post results."""
# Set up the coverage environment if it's needed.
cov_inst = None
if cov_env:
import coverage
try:
while True:
# Get the next test suite to run.
if test_suite == "STOP":
break
# Set up the test so that it plays nicely with tests
# running in other processes.
# Let the master process know that we have this test
# suite and we're about to start running it.
b = baseline.ReadOnlyBaseLine(
b.load()
# Build a _Pkg5TestResult object to use for this test.
show_on_expected_fail, a, cov)
try:
except TestStopException:
pass
# Pull in the information stored in places other than
# the _Pkg5TestResult that we need to send back to the
# master process.
if g_debug_output:
finally:
if cov_inst:
"""TestRunner for test suites that we want to be able to compare
against a result baseline."""
baseline = None
coverage=None,
"""Set up the test runner"""
# output is one of OUTPUT_DOTS, OUTPUT_VERBOSE, OUTPUT_PARSEABLE
if not class_list and not method_list:
return
tot = 0
print("Tests run for '{0}' Suite, broken down by class:\n".format(
print("{0:>6.2f} {1}.{2}".format(
continue
print("\nTests run for '{0}' Suite, " \
assert suite_name
total = 0
# Calculate the new time estimates for each test suite method
# run in the last run.
/ 2.0
# For each test class, find the average time each test in the
# class takes to run.
total = 0
m_cnt = 0
# for c in class_tot:
if cname == "TOTAL":
continue
c_tot = 0
c_cnt = 0
if mname == "CLASS":
continue
c_tot += \
c_cnt += 1
"CLASS", c_avg)
c_avg) // 2
# Calculate the average per test, regardless of which test class
# or method is being run.
# Save the estimates to disk.
timing = {}
lst = []
suite_name = None
if not lst:
return
if self.timing_history:
try:
# simpleson module will produce str
# in Python 3, therefore fh.write()
# must support str input.
"w+") as fh:
except KeyboardInterrupt:
raise TestStopException()
if not self.timing_file:
return
try:
except KeyboardInterrupt:
raise TestStopException()
except Exception:
if opened:
# If there's an estimate for the method, use it. If no method
# estimate is available, fall back to the average time each test
# in this class takes, if it's available. If not class estimate
# is available, fall back to the average time for each test in
# the test suite.
if c in time_estimates[suite_name]:
procs, start_times):
"""Given the running and unfinished tests, estimate the amount
of time remaining before the remaining tests finish."""
secs = 0
long_pole = 0
for mod, c in test_classes:
if suite_name not in time_estimates:
return None
class_tot = 0
class_tot += \
time_estimates, suite_name, c,
# Some tests have been running for a while, adjust the
# remaining time using this info.
if (mod, c) in start_times:
quiet):
if quiet:
return
print("\t{0}\t{1}\t{2} {3}".format(
if remaining_time is not None:
print("Estimated time remaining {0:d} " \
comm):
if quiet:
return
if g_debug_output:
print("Finished {0} {1} in process {2}".format(
print("Total test classes:{0} Finished test "
"classes:{1} Running tests:{2}".format(
print("Total tests:{0} Tests run:{1} "
"Errors:{2} Failures:{3} Skips:{4}".format(
print("Estimated time remaining {0:d} "
"""Terminate all processes in this process's task. This
assumes that test suite is running in its own task which
run.py should ensure."""
print("All spawned processes should be terminated, now "
# Terminated test jobs may have mounted filesystems under their
# images and not told us about them, so we catch EBUSY, unmount,
# and keep trying.
retry = 0
try:
except OSError as e:
#
# seems to sporadically happen if we
# race with e.g. something shutting
# down which has a pid file; retry.
#
retry += 1
continue
e.filename])
# if the umount failed, bump retry so
# we won't be stuck doing this forever.
if ret != 0:
retry += 1
continue
else:
raise
else:
if not finished:
else:
"Run the given test case or test suite."
started_tests = {}
finished_tests = set()
total_tests = 0
test_map = {}
start_times = {}
suite_name = None
# test case setUp() may require running pkg commands
# so setup a fakeroot to run them from.
for t in suite_list:
if not t.tests:
continue
if suite_name is None:
else:
raise RuntimeError("tmp:{0} mod:{1} "
if jobs > 1:
t.debug_output = False
if not all_tests:
try:
except OSError as e:
raise
return result
assert suite_name is not None
p_dict = {}
try:
except KeyboardInterrupt:
try:
while all_tests - finished_tests:
remaining_time = None
if time_estimates:
remaining_time = \
raise RuntimeError("Got "
"unexpected start "
for n, r in \
raise RuntimeError("mismatch")
comm)
else:
raise RuntimeError("unexpected "
if self.bailonfail and \
raise TestStopException()
# Check to make sure that all processes are
# still running.
for i in p_dict:
if broken:
print("The following "
"processes have died, "
"terminating the others: {0}".format(
",".join([
raise TestStopException()
for p in p_dict:
except (KeyboardInterrupt, TestStopException):
except Exception as e:
print(e)
raise
finally:
try:
if run > 0:
"in {2:>.3f}s - skipped {3:d} tests.\n".format(
if result.wasSkipped() and \
"tests:\n")
"{0}: {1}\n".format(
if not result.wasSuccessful():
success))
failed))
errored))
finally:
if terminate:
return result
"""Test suite that extends unittest.TestSuite to handle persistent
tests. Persistent tests are ones that are able to only call their
every test case. Aside from actually running the test it defers the
majority of its work to unittest.TestSuite.
To make a test class into a persistent one, add this class
variable declaration:
persistent_setup = True
"""
# The site module deletes the function to change the
# default encoding so a forced reload of sys has to
# be done at least once.
raise TestStopException()
inst = None
tdf = None
try:
"persistent_setup", False)
except IndexError:
# No tests; that's ok.
return
# This is needed because the import of some modules (such as
# pygtk or pango) causes the default encoding for Python to be
# changed which can can cause tests to succeed when they should
# fail due to unicode issues:
if not default_utf8:
# Now reset to the default a standard Python
# distribution uses.
else:
def setUp_donothing():
pass
def tearDown_donothing():
pass
def setUp_dofail():
raise TestSkippedException(
"Persistent setUp Failed, skipping test.")
if persistent_setup:
# Save a reference to the tearDown func and neuter
# normal per-test-function teardown.
else:
try:
except KeyboardInterrupt:
except:
# XXX do cleanup?
# If the setUp function didn't work, then cause
# every test case to fail.
if setUpFailed:
else:
if result.shouldStop:
break
#
# Update test environment settings. We redo this
# before running each test case since previously
# executed test cases may have messed with these
# environment settings.
#
# Populate test with the data from the instance
# already constructed, but update the method name.
# We need to do this so that we have all the state
# that the object is populated with when setUp() is
# called (depot controller list, etc).
if persistent_setup:
else:
# If rtest is a copy of test, then we need to copy
# rtest's buffer back to test's so that it has the
# output from the run.
if persistent_setup:
if persistent_setup:
try:
except KeyboardInterrupt:
except:
# Try to ensure that all depots have been nuked.
t.base_port = p
t.ident = i
self.__debug_output = v
t.debug_output = v
return self.__debug_output
res = ""
res += "\n"
return res
for u in ["noaccess", "nobody"]:
try:
if uid_gid:
return operator.attrgetter(
return u
pass
raise RuntimeError("Unable to determine user for su.")
str = "During this test, a depot Traceback was detected.\n"
str += "Log file output is:\n"
return str
str = ""
return str
str = ""
return str
image_files = []
self.__imgs_path = {}
# ii is the image index
return
return self.__imgs_index
if ii != None:
# for backward compatibilty
"""Given a path relative to root, return the absolute path of
the item in the image."""
if not cmd_path:
if not img_path:
return res
return
# Set up the trust anchor directory
for f in self.image_files:
"""A convenience wrapper for callers that only need basic image
creation functionality. This wrapper creates a full (as opposed
to user) image using the pkg.client.api and returns the related
API object."""
if img_path is None:
if destroy:
**kwargs)
return api_inst
"""Executes pkg(1) client to create a full (as opposed to user)
image; returns exit code of client or raises an exception if
exit code doesn't match 'exit' or equals 99."""
prefix = "test"
if repourl:
return retcode
# the currently selected image is the source
# create an empty destination image
# reactivate the source image
# populate the destination image
if img_path is None:
# Make sure we're not in the image.
else:
if not cmd_path:
"image-create" not in cmdstr):
DebugValues["smf_cmds_dir"])))
else:
if assert_solution:
# Ensure solver never fails with 'No solution' by
# default to prevent silent failures for the wrong
# reason.
msg="Solver could not find solution for "
"operation; set assert_solution=False if "
"this is expected when calling pkg().")
return rval
"""Wraps self.pkg(..) and checks that the 'verify' command run
does not contain the string 'Unexpected Exception', indicating
something has gone wrong during package verification."""
"Unexpected errors encountered while verifying.")
return res
env_arg=None):
ops = ""
if "-R" not in args:
env_arg=None):
env_arg=None):
if testrc:
else:
args = []
if server_url:
if command:
env_arg=None):
if debug_hash:
else:
debug_arg = ""
# Always add the current ignored_deps dir path.
g_pkg_path, "usr/share/pkg/ignored_deps"))
env_arg=None, debug_hash=None):
args = []
if depot_url:
if debug_hash:
if command:
debug_hash=None):
"ch1_ta3_cert.pem")
)
debug_hash=None):
args = []
if allow_timestamp:
if depot_url:
# debug_hash lets us choose the type of hash attributes that
# should be added to this package on publication. Valid values
# are: sha1, sha256, sha1+sha256, sha512t_256, sha1+sha512t_256
if debug_hash:
if command:
assert arr
# retcode != 0 will be handled below
published = None
for l in out.splitlines():
if l.startswith("pkg:/"):
published = l
break
if retcode == 99:
debug_hash=None):
""" Send a series of packaging commands; useful for quickly
doing a bulk-load of stuff into the repo. All commands are
expected to work; if not, the transaction is abandoned. If
'exit' is set, then if none of the actions triggers that
exit code, an UnexpectedExitCodeException is raised.
A list containing the fmris of any packages that were
published as a result of the commands executed will be
returned; it will be empty if none were. """
extra_opts = []
if no_catalog:
plist = []
try:
accumulate = []
current_fmri = None
retcode = None
# pkgsend_bulk can't be used w/ import or
# generate.
"pkgsend_bulk cannot be used with import"
"pkgsend_bulk cannot be used with generate"
if line == "":
continue
"Missing open in pkgsend string")
continue
if current_fmri: # send any content seen so far (can be 0)
for l in accumulate:
"{0}\n".format(l)))
if su_wrap:
try:
current_fmri = None
accumulate = []
# Various tests rely on the
# ability to specify version
# down to timestamp for ease
# of testing or because they're
# actually testing timestamp
# package behaviour.
except:
raise
# If no explicit pkg.fmri set
# action was found, add one.
"name=pkg.fmri value={0}".format(
except UnexpectedExitCodeException as e:
raise
return plist
ops = ""
if "-R" not in args:
if "-c" not in args:
"sysrepo_cache"))
if "-l" not in args:
"sysrepo_logs"))
if "-r" not in args:
"sysrepo_runtime"))
if "-t" not in args:
if env_arg is None:
env_arg = {}
"""A convenient method to cause test execution to pause for
up to 'sleeptime' seconds, which can be helpful during testcase
development. sleeptime defaults to 3 hours."""
if show_stack:
"""Run pkg.depot-config, with command line arguments in args.
If fill_missing_args is set, we use default settings for several
arguments to point to template, logs, cache and proto areas
within our test root."""
args += " -S "
"depot_cache"))
if "-l" not in args:
"depot_logs"))
if "-r" not in args:
"depot_runtime"))
if "-T" not in args:
DebugValues["smf_cmds_dir"])
if env_arg is None:
env_arg = {}
"""Copies the packages from the src repository to a new
destination repository that will be created at dest. In
addition, any packages from the src_pub will be assigned
to the dest_pub during the copy. The new repository will
not have a catalog or search indices, so a depot server
pointed at the new repository must be started with the
--rebuild option.
"""
# Preserve destination repository's configuration if it exists.
dest_cfg_data = None
dest_cfg_data = f.read()
# Ensure config is written back out.
if dest_cfg_data:
# Now copy each manifest and replace any references to
# the old publisher with that of the new publisher as
# they are copied.
stem)
# Ensure destination manifest directory
# exists.
mname), "r")
mname), "w")
for l in msrc:
continue
nl = l
continue
# Skip the catalog, index, and pkg
# directories as they will be copied
# manually.
"pkg", "tmp", "trans"):
continue
if entry != "pkg":
continue
"""Returns the path to the manifest cache directory for the
given fmri."""
# Allow callers to not specify a fully-qualified FMRI
# if it can be asssumed which publisher likely has
# the package.
pubs = [
p.prefix
]
if not pubs:
# Include prefixes of publishers of installed
# packages that are no longer configured.
"""Returns the path to the manifest for the given fmri."""
# Allow callers to not specify a fully-qualified FMRI
# if it can be asssumed which publisher likely has
# the package.
pubs = [
p.prefix
]
if not pubs:
# Include prefixes of publishers of installed
# packages that are no longer configured.
"""Retrieves the client's cached copy of the manifest for the
given package FMRI and returns it as a string. Callers are
responsible for all error handling."""
return f.read()
"""Overwrites the client's cached copy of the manifest for the
given package FMRI using the provided string. Callers are
responsible for all error handling."""
# Dump the manifest directories for the package to ensure any
# cached information related to it is gone.
# Finally, write the new manifest.
"""Used to verify that the target item's mode, attrs, timestamp,
etc. match as expected. The actual"""
return
if not target:
# Verify owner.
# Verify group.
# Verify mode.
""" Run a html file specified by fname through a html validator
(tidy). The drop_prop_attrs parameter can be used to ignore
proprietary attributes which would otherwise make tidy fail.
"""
if drop_prop_attrs:
""" Convenience routine to help subclasses create a package
repository. Returns a pkg.server.repository.Repository
object. """
# Note that this must be deferred until after PYTHONPATH
# is set up.
try:
except sr.RepositoryExistsError:
# Already exists.
return repo
""" Convenience routine to help subclasses retrieve a
pkg.server.repository.Repository object for a given
path. """
# Note that this must be deferred until after PYTHONPATH
# is set up.
""" Convenience routine to help subclasses prepare
depots. Returns a depotcontroller. """
# Note that this must be deferred until after PYTHONPATH
# is set up.
"usr/lib/pkg.depotd"))
for f in debug_features:
for section in properties:
if refresh_index:
if start:
# If the caller requested the depot be started, then let
# the depot process create the repository.
try:
except Exception as e:
"depot!: {0}".format(e))
raise
else:
# Otherwise, create the repository with the assumption
# that the caller wants that at the least, but doesn't
# need the depot server (yet).
return dc
"""Wait for the specified repository to complete its current
operations before continuing."""
check_interval = 0.20
if repo_status == "online":
{}).values():
break
else:
# All repository stores were ready.
if not ready:
else:
break
if not ready:
raise RuntimeError("Repository readiness "
"timeout exceeded.")
continue
continue
continue
**kwargs):
plan = None
if plan is not None:
continue
# update license status
plan.get_licenses():
if noexecute:
return
**kwargs):
continue
if noexecute:
return
continue
if noexecute:
return
continue
if noexecute:
return
**kwargs):
continue
if noexecute:
return
continue
**kwargs):
**kwargs):
continue
if noexecute:
return
continue
try:
except apx.WrapSuccessfulIndexingException:
if not catch_wsie:
raise
"""Return the inode number of a file in the image."""
"""Return the size of a file in the image."""
"""Change the mode of a file in the image."""
"""Assert the existence of a file in the image."""
"""Assert the existence of a directory in the image."""
try:
except OSError as e:
else:
raise
if mode is not None:
if owner is not None:
if group is not None:
"""Assert the non-existence of a file in the image."""
""""Assert that files are there in the image."""
for p in paths:
else:
else:
self.file_exists(p)
"""Assert that files are all missing in the image."""
for p in paths:
"""Remove a file in the image."""
"""Assert the existence of strings provided in a file in the
image. The counting of appearances is line based. Repeated
string on the same line will be count once."""
# Initialize a dict for counting appearances.
sdict = {}
for s in strings:
sdict[s] = appearances
try:
except:
"File {0} does not exist or contain {1}".format(
for line in f:
for k in sdict:
if k in line:
sdict[k] -= 1
# If all counts become < 0, we know we have found all
# occurrences for all strings.
f.close()
break
else:
f.close()
"""Assert the non-existence of strings in a file in the image.
"""
for line in f:
f.close()
else:
f.close()
"""Append a line to a file in the image."""
if not dest_dir:
for c in certs:
for p in paths:
p = p[1:]
continue
f.write("\ncontents\n")
image_count=1):
i = n + 1
try:
except OSError as e:
raise e
"depot_logfile{0:d}".format(i))
# We pick an arbitrary base port. This could be more
# automated in the future.
""" Scan logpath looking for tracebacks.
Raise a DepotTracebackException if one is seen.
"""
print("Ctrl-C: I'm killing depots, please wait.\n",
print(self)
try:
check_dc = []
continue
status = 0
try:
except Exception:
pass
if status:
try:
except Exception:
pass
finally:
raise KeyboardInterrupt("Ctrl-C while killing depots.")
if result is None:
"""A TestCase that uses one or more Apache instances in the course of
its work, along with potentially one or more DepotControllers.
"""
"""Registers an ApacheController with this TestCase.
We include this method here to make it easier to kill any
instances of Apache that were left floating around at the end
of the test.
We enforce the use of this method in
<ApacheController>.start() by refusing to start instances until
they are registered, which makes the test suite as a whole more
resilient, when setting up and tearing down test classes."""
# registering an Apache controller that is already
# registered causes us to kill the existing controller
# first.
try:
except Exception as e:
try:
except Exception as e:
pass
"""If we only use a single ApacheController, self.ac will
return that controller, otherwise we return None."""
else:
return None
try:
finally:
name))
try:
except Exception as e :
try:
except Exception as e:
"apache instance {0}. This "
"could cause subsequent "
# ac is a readonly property which returns a registered ApacheController
# provided there is exactly one registered, for convenience of writing
# test cases.
# Tests in this suite use the read only data directory.
# The value for ssl_ca_file is pulled from DebugValues because
# ssl_ca_file needs to be set there so the api object calls work
# as desired.
# The value for ssl_ca_file is pulled from DebugValues because
# ssl_ca_file needs to be set there so the api object calls work
# as desired.
# The value for ssl_ca_file is pulled from DebugValues because
# ssl_ca_file needs to be set there so the api object calls work
# as desired.
# The value for ssl_ca_file is pulled from DebugValues because
# ssl_ca_file needs to be set there so the api object calls work
# as desired.
if not dest_dir:
for c in certs:
name)
# We only have 5 usable CA certs and there are not many usecases
# for setting up more than 5 different SSL-secured depots.
# Maintains a mapping of which TA is used for which publisher
self.pub_ta_map = {}
# Set up the directories that apache needs.
"apache_logs")
"apache_content")
"apache-serve")
# Choose ports for apache to run on.
# Set up the paths to the certificates that will be needed.
"signing_certs", "produced")
"code_signing_certs")
"chain_certs")
"publisher_cas")
"inter_certs")
"trust_anchors")
location_tags = ""
# Usable CA certs are ta6 to ta11 with the exception of ta7.
# We already checked that not more than 5 publishers have been
# requested.
count = 6
# Create a <Location> tag for each publisher. The server
# path is set to the publisher name.
if count == 7:
# TA7 needs password to unlock cert, don't use
count += 1
"prefix")
loc_dict = {
"server-path":dc_pub,
"ssl-special":"%{SSL_CLIENT_I_DN_OU}",
}
count += 1
conf_dict = {
"common_log_format": "%h %l %u %t \\\"%r\\\" %>s %b",
"cs1_ta7_cert.pem"),
"cs1_ta7_key.pem"),
"location-tags":location_tags,
}
"https.conf")
https_conf = """\
# Configuration and logfile names: If the filenames you specify for many
# of the server's control files begin with "/" (or "drive:/" for Win32), the
# server will use that explicit path. If the filenames do *not* begin
# with "/", the value of ServerRoot is prepended -- so "logs/access_log"
# will be interpreted as "/logs/access_log".
#
# ServerRoot: The top of the directory tree under which the server's
# configuration, error, and log files are kept.
#
# Do not add a slash at the end of the directory path. If you point
# ServerRoot at a non-local disk, be sure to point the LockFile directive
# at a local disk. If you wish to share the same ServerRoot for multiple
# httpd daemons, you will need to change at least LockFile and PidFile.
#
PidFile "{pidfile}"
#
# ports, instead of the default. See also the <VirtualHost>
# directive.
#
# Change this to Listen on specific IP addresses as shown below to
# prevent Apache from glomming onto all bound IP addresses.
#
Listen 0.0.0.0:{https_port}
# We also make ourselves a general-purpose proxy. This is not needed for the
# SSL reverse-proxying to the pkg.depotd, but allows us to test that pkg(1)
# can communicate to HTTPS origins using a proxy.
Listen 0.0.0.0:{proxy_port}
Listen 0.0.0.0:{bad_proxy_port}
#
# Dynamic Shared Object (DSO) Support
#
# To be able to use the functionality of a module which was built as a DSO you
# have to place corresponding `LoadModule' lines at this location so the
# directives contained in it are actually available _before_ they are used.
# Statically compiled modules (those listed by `httpd -l') do not need
# to be loaded here.
#
LoadModule access_compat_module libexec/mod_access_compat.so
LoadModule alias_module libexec/mod_alias.so
LoadModule authn_core_module libexec/mod_authn_core.so
LoadModule authz_core_module libexec/mod_authz_core.so
LoadModule authz_host_module libexec/mod_authz_host.so
LoadModule cache_module libexec/mod_cache.so
LoadModule deflate_module libexec/mod_deflate.so
LoadModule dir_module libexec/mod_dir.so
LoadModule env_module libexec/mod_env.so
LoadModule filter_module libexec/mod_filter.so
LoadModule headers_module libexec/mod_headers.so
LoadModule log_config_module libexec/mod_log_config.so
LoadModule mime_module libexec/mod_mime.so
LoadModule mpm_worker_module libexec/mod_mpm_worker.so
LoadModule rewrite_module libexec/mod_rewrite.so
LoadModule ssl_module libexec/mod_ssl.so
LoadModule proxy_module libexec/mod_proxy.so
LoadModule proxy_connect_module libexec/mod_proxy_connect.so
LoadModule proxy_http_module libexec/mod_proxy_http.so
LoadModule unixd_module libexec/mod_unixd.so
<IfModule unixd_module>
#
# If you wish httpd to run as a different user or group, you must run
# httpd as root initially and it will switch.
#
# It is usually good practice to create a dedicated user and group for
# running httpd, as with most system services.
#
User webservd
Group webservd
</IfModule>
# 'Main' server configuration
#
# The directives in this section set up the values used by the 'main'
# server, which responds to any requests that aren't handled by a
# <VirtualHost> definition. These values also provide defaults for
# any <VirtualHost> containers you may define later in the file.
#
# All of these directives may appear inside <VirtualHost> containers,
# in which case these default settings will be overridden for the
# virtual host being defined.
#
#
# ServerName gives the name and port that the server uses to identify itself.
# This can often be determined automatically, but we recommend you specify
# it explicitly to prevent problems during startup.
#
# If your host doesn't have a registered DNS name, enter its IP address here.
#
ServerName 127.0.0.1
#
# DocumentRoot: The directory out of which you will serve your
# documents. By default, all requests are taken from this directory, but
# symbolic links and aliases may be used to point to other locations.
#
DocumentRoot "/"
#
# Each directory to which Apache has access can be configured with respect
# directory (and its subdirectories).
#
# First, we configure the "default" to be a very restrictive set of
# features.
#
<Directory />
Options None
AllowOverride None
Require all denied
</Directory>
#
# Note that from this point forward you must specifically allow
# particular features to be enabled - so if something's not working as
# you might expect, make sure that you have specifically enabled it
# below.
#
#
# This should be changed to whatever you set DocumentRoot to.
#
#
# DirectoryIndex: sets the file that Apache will serve if a directory
# is requested.
#
<IfModule dir_module>
DirectoryIndex index.html
</IfModule>
#
# The following lines prevent .htaccess and .htpasswd files from being
# viewed by Web clients.
#
<FilesMatch "^\.ht">
Require all denied
</FilesMatch>
#
# ErrorLog: The location of the error log file.
# If you do not specify an ErrorLog directive within a <VirtualHost>
# container, error messages relating to that virtual host will be
# logged here. If you *do* define an error logfile for a <VirtualHost>
# container, that host's errors will be logged there and not here.
#
ErrorLog "{log_locs}/error_log"
#
# LogLevel: Control the number of messages logged to the error_log.
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
#
LogLevel debug
<IfModule log_config_module>
#
# The following directives define some format nicknames for use with
# a CustomLog directive (see below).
#
LogFormat "{common_log_format}" common
LogFormat "PROXY {common_log_format}" proxylog
#
# The location and format of the access logfile (Common Logfile Format).
# If you do not define any access logfiles within a <VirtualHost>
# container, they will be logged here. Contrariwise, if you *do*
# define per-<VirtualHost> access logfiles, transactions will be
# logged therein and *not* in this file.
#
CustomLog "{log_locs}/access_log" common
</IfModule>
<IfModule mime_module>
#
# TypesConfig points to the file containing the list of mappings from
# filename extension to MIME-type.
#
#
# AddType allows you to add to or override the MIME configuration
# file specified in TypesConfig for specific file types.
#
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
# Add a new mime.type for .p5i file extension so that clicking on
# this file type on a web page launches PackageManager in a Webinstall mode.
AddType application/vnd.pkg5.info .p5i
</IfModule>
#
# Note: The following must must be present to support
# but a statically compiled-in mod_ssl.
#
<IfModule ssl_module>
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
</IfModule>
<VirtualHost 0.0.0.0:{https_port}>
AllowEncodedSlashes On
ProxyRequests Off
MaxKeepAliveRequests 10000
SSLEngine On
# Cert paths
SSLCertificateFile {server-ssl-cert}
SSLCertificateKeyFile {server-ssl-key}
# Combined product CA certs for client verification
SSLCACertificateFile {server-ca-cert}
SSLVerifyClient require
{location-tags}
</VirtualHost>
#
# We configure this Apache instance as a general-purpose HTTP proxy, accepting
# requests from localhost, and allowing CONNECTs to our HTTPS port
#
<VirtualHost 0.0.0.0:{proxy_port}>
<Proxy *>
Require local
</Proxy>
AllowCONNECT {https_port}
ProxyRequests on
CustomLog "{log_locs}/proxy_access_log" proxylog
</VirtualHost>
<VirtualHost 0.0.0.0:{bad_proxy_port}>
<Proxy *>
Require local
</Proxy>
# We purposely prevent this proxy from being able to connect to our SSL
# port, making sure that when we point pkg(1) to this bad proxy, operations
# will fail - the following line is commented out:
# AllowCONNECT {https_port}
ProxyRequests on
CustomLog "{log_locs}/badproxy_access_log" proxylog
</VirtualHost>
"""
loc_tag = """
<Location /{server-path}>
SSLVerifyDepth 1
# The client's certificate must pass verification, and must have
# a CN which matches this repository.
SSLRequire ( {ssl-special} =~ m/{server-ca-taname}/ )
# set max to number of threads in depot
ProxyPass {proxied-server} nocanon max=500
</Location>
"""
else:
return None
# dc is a readonly property which is an alias for self.dcs[1],
# for convenience of writing test cases.
""" A class which allows manipulation of the image directory that
SingleDepotTestCase creates. Specifically, it supports removing one
or more of the files or subdirectories inside an image (publisher,
cfg_cache, etc...) in a controlled way.
To add a new directory or file to be corrupted, it will be necessary
to update corrupt_image_create to recognize a new option in config
and perform the appropriate action (removing the directory or file
for example).
"""
self.__imgs_path_backup = {}
if ii != None:
""" Creates two levels of directories under the original image
directory. In the first level (called bad), it builds a "corrupt
image" which means it builds subdirectories the subdirectories
specified by subdirs (essentially determining whether a user
image or a full image will be built). It populates these
subdirectories with a partial image directory stucture as
specified by config. As another subdirectory of bad, it
creates a subdirectory called final which represents the
directory the command was actually run from (which is why
img_path is set to that location). Existing image destruction
was made optional to allow testing of two images installed next
to each other (a user and full image created in the same
directory for example). """
if destroy:
for s in subdirs:
elif s == ".org.opensolaris,pkg":
else:
raise RuntimeError("Got unknown subdir option:"
"{0}\n".format(s))
" " + cmdline
# This is where the actual corruption of the
# image takes place. A normal image was created
# above and this goes in and removes critical
# directories and files.
if "publisher_absent" in config or \
"publisher_empty" in config:
if "known_absent" in config or \
"known_empty" in config:
"known"))
if "known_empty" in config:
if "publisher_empty" in config:
if "cfg_cache_absent" in config:
if "index_absent" in config:
"index"))
# Make find root start at final. (See the doc string for
# more explanation.)
return cmd_path
def debug(s):
s = str(s)
for x in s.splitlines():
if g_debug_output:
def mkdir_eexist_ok(p):
try:
except OSError as e:
raise e
if dv_keep == None:
dv_keep = []
dv_saved = {}
# save some DebugValues settings
# clear any existing DebugValues settings
# clear misc environment variables
for e in ["PKG_CMDPATH"]:
# Set image path to a path that's not actually an
# image to force failure of tests that don't
# explicitly provide an image root either through the
# default behaviour of the pkg() helper routine or
# another method.
# Test suite should never attempt to access the
# live root image.
# Pkg interfaces should never know they are being
# run from within the test suite.
# verify PlanDescription serialization and that the PlanDescription
# isn't modified while we're preparing to for execution.
# Pretend that we're being run from the fakeroot image.
assert pkg_cmdpath != "TOXIC"
# Update the path to smf commands
# always get detailed data from the solver
def fakeroot_create():
try:
except OSError as e:
pass
else:
# fakeroot already exists
raise RuntimeError("The fakeroot shouldn't already exist.\n"
# when creating the fakeroot we want to make sure pkg doesn't
# touch the real root.
#
# When accessing images via the pkg apis those apis will try
# to access the image containing the command from which the
# apis were invoked. Normally when running the test suite the
# command is run.py in a developers workspace, and that
# workspace lives in the root image. But accessing the root
# image during a test suite run is verboten. Hence, here we
# create a temporary image from which we can run the pkg
# command.
#
# create directories
#
# put a copy of the pkg command in our fake root directory.
# we do this because when recursive linked image commands are
# run, the pkg interfaces may fork and exec additional copies
# of pkg(1), and in this case we want them to run the copy of
# pkg from the fake root.
#
return fakeroot, fakeroot_cmdpath
try:
except ex_type as e:
print(str(e))
if not eval_ex_func(e):
raise
else:
raise RuntimeError("Function did not raise exception.")
pass
"""
The 'conf' parameter is a path to a httpd.conf file. The 'port'
parameter is a port to run on. The 'work_dir' is a temporary
directory to store runtime state. The 'testcase' parameter is
the ApacheDepotTestCase to use when writing output. The 'https'
parameter is a boolean indicating whether this instance expects
to be contacted via https or not.
"""
self.__repo_hdl = None
if not testcase:
raise RuntimeError("No testcase parameter specified")
prefix = "http"
if https:
prefix = "https"
return self.__conf_path
try:
except HTTPError as e:
return True
return False
except URLError as e:
return True
return False
return True
# An attempt to start an ApacheController that has not
# been registered can result in it not getting cleaned
# up properly when the test completes, which can cause
# other tests to fail. We don't allow that to happen.
raise RuntimeError(
"This ApacheController has not been registered with"
" the ApacheDepotTestCase {0} using "
if self._network_ping():
raise ApacheStateException("A depot (or some " +
"other network process) seems to be " +
try:
# change the state so that we try to do work in
# self.stop() in the face of a False result from
# is_alive()
if self.__repo_hdl is None:
raise ApacheStateException("Could not start "
"apache")
check_interval = 0.20
if rc is not None:
raise ApacheStateException("Apache "
"exited unexpectedly while "
break
raise ApacheStateException("Apache did not "
"respond to repeated attempts to make "
"contact")
except KeyboardInterrupt:
if self.__repo_hdl:
raise
if not self.__repo_hdl:
return
try:
finally:
try:
except OSError:
pass
return
"stop"]
try:
if stop_errout != "":
if stop_output != "":
if stop_retcode != 0:
else:
# Ensure that the apache process gets shutdown
check_interval = 0.20
if rc is not None:
break
if not stopped:
# retrieve output from the apache process we've just
# stopped
except KeyboardInterrupt:
raise
pass
print("Ctrl-C: I'm killing depots, please wait.\n",
print(self)
""" First, check that the depot process seems to be alive.
Then make a little HTTP request to see if the depot is
responsive to requests """
if self.__repo_hdl == None:
return False
if status != None:
return False
return self._network_ping()
try:
except HTTPError as e:
return True
return False
except URLError:
return False
return True
try:
# Ping the versions URL, rather than the default /
# so that we don't initialize the BUI code yet.
# Disable SSL peer verification, we just want to check
# if the depot is running.
except HTTPError as e:
return True
return False
except URLError:
return False
return True