106N/A

106N/A

106N/Afrom collections import namedtuple

106N/Aimport copy

1247N/Aimport errno

618N/Aimport hashlib

106N/Aimport os

106N/Aimport tempfile

844N/Afrom itertools import groupby, chain, repeat

1247N/A

1247N/Aimport pkg.actions as actions

1273N/Aimport pkg.client.api_errors as apx

106N/Aimport pkg.misc as misc

3661N/Aimport pkg.portable as portable

3661N/Aimport pkg.variant as variant

3778N/A

3778N/Afrom pkg.misc import EmptyDict, EmptyI, expanddirs, PKG_FILE_MODE, PKG_DIR_MODE

3778N/Afrom pkg.actions.attribute import AttributeAction

3996N/A

4561N/AManifestDifference = namedtuple("ManifestDifference", "added changed removed")

4561N/A

4561N/Aclass Manifest(object):

4561N/A """A Manifest is the representation of the actions composing a specific

1247N/A

106N/A def __init__(self, pfmri=None):

106N/A self.fmri = pfmri

1247N/A

1247N/A self.actions = []

1247N/A self.actions_bytype = {}

247N/A self.variants = {} # variants seen in package

1874N/A self.facets = {} # facets seen in package

106N/A self.attributes = {} # package-wide attributes

3996N/A self.signatures = EmptyDict

4237N/A

def __str__(self):

r = ""

if "pkg.fmri" not in self.attributes and self.fmri != None:

r += "set name=pkg.fmri value=%s\n" % self.fmri

for act in sorted(self.actions):

r += "%s\n" % act

return r

def as_lines(self):

"""A generator function that returns the unsorted manifest

if "pkg.fmri" not in self.attributes and self.fmri != None:

yield "set name=pkg.fmri value=%s\n" % self.fmri

for act in self.actions:

yield "%s\n" % act

def tostr_unsorted(self):

return "".join((l for l in self.as_lines()))

def difference(self, origin, origin_exclude=EmptyI,

self_exclude=EmptyI):

"""Return three lists of action pairs representing origin and

# XXX Do we need to find some way to assert that the keys are

# all unique?

if isinstance(origin, EmptyFactoredManifest):

# No origin was provided, so nothing has been changed or

# removed; only added. In addition, this doesn't need

# to be sorted since the caller likely already does

# (such as pkgplan/imageplan).

return ManifestDifference(

[(None, a) for a in self.gen_actions(self_exclude)],

[], [])

def hashify(v):

"""handle key values that may be lists"""

if isinstance(v, list):

return frozenset(v)

else:

return v

sdict = dict(

((a.name, hashify(a.attrs.get(a.key_attr, id(a)))), a)

for a in self.gen_actions(self_exclude)

)

odict = dict(

((a.name, hashify(a.attrs.get(a.key_attr, id(a)))), a)

for a in origin.gen_actions(origin_exclude)

)

sset = set(sdict.keys())

oset = set(odict.keys())

added = [(None, sdict[i]) for i in sset - oset]

removed = [(odict[i], None) for i in oset - sset]

changed = [

(odict[i], sdict[i])

for i in oset & sset

if odict[i].different(sdict[i])

]

# XXX Do changed actions need to be sorted at all? This is

# likely to be the largest list, so we might save significant

# time by not sorting. Should we sort above? Insert into a

# sorted list?

# singlesort = lambda x: x[0] or x[1]

addsort = lambda x: x[1]

remsort = lambda x: x[0]

removed.sort(key = remsort, reverse = True)

added.sort(key = addsort)

changed.sort(key = addsort)

return ManifestDifference(added, changed, removed)

@staticmethod

def comm(*compare_m):

"""Like the unix utility comm, except that this function

# construct list of dictionaries of actions in each

# manifest, indexed by unique keys

m_dicts = [

dict(

((a.name, a.attrs.get(a.key_attr, id(a))), a)

for a in m.actions)

for m in compare_m

]

# construct list of key sets in each dict

#

m_sets = [

set(m.keys())

for m in m_dicts

]

common_keys = reduce(lambda a, b: a & b, m_sets)

# determine which common_keys have common actions

for k in common_keys.copy():

for i in range(len(m_dicts) - 1):

if m_dicts[i][k].different(

m_dicts[i + 1][k]):

common_keys.remove(k)

break

return tuple(

[

[m_dicts[i][k] for k in m_sets[i] - common_keys]

for i in range(len(m_dicts))

]

+

[

[ m_dicts[0][k] for k in common_keys ]

]

)

def combined_difference(self, origin, ov=EmptyI, sv=EmptyI):

"""Where difference() returns three lists, combined_difference()

return list(chain(*self.difference(origin, ov, sv)))

def humanized_differences(self, other, ov=EmptyI, sv=EmptyI):

"""Output expects that self is newer than other. Use of sets

l = self.difference(other, ov, sv)

out = ""

for src, dest in chain(*l):

if not src:

out += "+ %s\n" % str(dest)

elif not dest:

out += "- %s\n" + str(src)

else:

out += "%s -> %s\n" % (src, dest)

return out

def gen_actions(self, excludes=EmptyI):

"""Generate actions in manifest through ordered callable list"""

for a in self.actions:

for c in excludes:

if not c(a):

break

else:

yield a

def gen_actions_by_type(self, atype, excludes=EmptyI):

"""Generate actions in the manifest of type "type"

for a in self.actions_bytype.get(atype, []):

for c in excludes:

if not c(a):

break

else:

yield a

def gen_key_attribute_value_by_type(self, atype, excludes=EmptyI):

"""Generate the value of the key atrribute for each action

return (

a.attrs.get(a.key_attr)

for a in self.gen_actions_by_type(atype, excludes)

)

def duplicates(self, excludes=EmptyI):

"""Find actions in the manifest which are duplicates (i.e.,

def fun(a):

"""Return a key on which actions can be sorted."""

return a.name, a.attrs.get(a.key_attr, id(a))

alldups = []

acts = [a for a in self.gen_actions(excludes)]

for k, g in groupby(sorted(acts, key=fun), fun):

glist = list(g)

dups = set()

for i in range(len(glist) - 1):

if glist[i].different(glist[i + 1]):

dups.add(glist[i])

dups.add(glist[i + 1])

if dups:

alldups.append((k, dups))

return alldups

def __content_to_actions(self, content):

accumulate = ""

lineno = 0

errors = []

if isinstance(content, basestring):

# Get an iterable for the string.

content = content.splitlines()

for l in content:

lineno += 1

l = l.lstrip()

if l.endswith("\\"): # allow continuation chars

accumulate += l[0:-1] # elide backslash

continue

elif accumulate:

l = accumulate + l

accumulate = ""

if not l or l[0] == "#": # ignore blank lines & comments

continue

try:

yield actions.fromstr(l)

except actions.ActionError, e:

# Accumulate errors and continue so that as

# much of the action data as possible can be

# parsed.

e.fmri = self.fmri

e.lineno = lineno

errors.append(e)

if errors:

raise apx.InvalidPackageErrors(errors)

def set_content(self, content=None, excludes=EmptyI, pathname=None,

signatures=False):

"""Populate the manifest with actions.

assert content is not None or pathname is not None

assert not (content and pathname)

self.actions = []

self.actions_bytype = {}

self.variants = {}

self.facets = {}

self.attributes = {}

# So we could build up here the type/key_attr dictionaries like

# sdict and odict in difference() above, and have that be our

# main datastore, rather than the simple list we have now. If

# we do that here, we can even assert that the "same" action

# can't be in a manifest twice. (The problem of having the same

# action more than once in packages that can be installed

# together has to be solved somewhere else, though.)

if pathname:

try:

with open(pathname, "rb") as mfile:

content = mfile.read()

except EnvironmentError, e:

raise apx._convert_error(e)

if isinstance(content, basestring):

if signatures:

# Generate manifest signature based upon

# input content, but only if signatures

# were requested.

self.signatures = {

"sha-1": self.hash_create(content)

}

content = self.__content_to_actions(content)

for action in content:

self.add_action(action, excludes)

def exclude_content(self, excludes):

"""Remove any actions from the manifest which should be

self.set_content(content=self.actions, excludes=excludes)

def add_action(self, action, excludes):

"""Performs any needed transformations on the action then adds

# XXX handle legacy transition issues; not needed after

# 2009.06 release & republication are complete.

if "opensolaris.zone" in action.attrs and \

"variant.opensolaris.zone" not in action.attrs:

action.attrs["variant.opensolaris.zone"] = \

action.attrs["opensolaris.zone"]

if action.name == "set" and action.attrs["name"] == "authority":

# Translate old action to new.

action.attrs["name"] = "publisher"

if action.attrs.has_key("path"):

np = action.attrs["path"].lstrip(os.path.sep)

action.attrs["path"] = np

if not action.include_this(excludes):

return

self.actions.append(action)

self.actions_bytype.setdefault(action.name, []).append(action)

# add any set actions to attributes

if action.name == "set":

self.fill_attributes(action)

# append any variants and facets to manifest dict

v_list, f_list = action.get_varcet_keys()

if v_list or f_list:

for v, d in zip(v_list, repeat(self.variants)) \

+ zip(f_list, repeat(self.facets)):

if v not in d:

d[v] = set([action.attrs[v]])

else:

d[v].add(action.attrs[v])

def fill_attributes(self, action):

"""Fill attribute array w/ set action contents."""

try:

keyvalue = action.attrs["name"]

if keyvalue == "fmri":

keyvalue = "pkg.fmri"

if keyvalue not in self.attributes:

self.attributes[keyvalue] = \

action.attrs["value"]

except KeyError: # ignore broken set actions

pass

@staticmethod

def search_dict(file_path, excludes, return_line=False,

log=None):

"""Produces the search dictionary for a specific manifest.

if log is None:

log = lambda x: None

file_handle = file(file_path, "rb")

cur_pos = 0

line = file_handle.readline()

action_dict = {}

def __handle_list(lst, cp):

"""Translates what actions.generate_indices produces

for action_name, subtype, tok, full_value in lst:

if action_name == "set":

if full_value is None:

full_value = tok

else:

if full_value is None:

full_value = subtype

if full_value is None:

full_value = action_name

if isinstance(tok, list):

__handle_list([

(action_name, subtype, t,

full_value)

for t in tok

], cp)

else:

if (tok, action_name, subtype,

full_value) in action_dict:

action_dict[(tok, action_name,

subtype, full_value)

].append(cp)

else:

action_dict[(tok, action_name,

subtype, full_value)] = [cp]

while line:

l = line.strip()

if l and l[0] != "#":

try:

action = actions.fromstr(l)

except actions.ActionError, e:

log((_("%(fp)s:\n%(e)s") %

{ "fp": file_path, "e": e }))

else:

if action.include_this(excludes):

if action.attrs.has_key("path"):

np = action.attrs["path"].lstrip(os.path.sep)

action.attrs["path"] = \

np

try:

inds = action.generate_indices()

except KeyError, k:

log(_("%(fp)s contains "

"an action which is"

" missing the "

"expected attribute"

": %(at)s.\nThe "

"action is:"

"%(act)s") %

{

"fp": file_path,

"at": k.args[0],

"act":l

})

else:

arg = cur_pos

if return_line:

arg = l

__handle_list(inds, arg)

cur_pos = file_handle.tell()

line = file_handle.readline()

file_handle.close()

return action_dict

@staticmethod

def hash_create(mfstcontent):

"""This method takes a string representing the on-disk

sha_1 = hashlib.sha1()

if isinstance(mfstcontent, unicode):

# Byte stream expected, so pass encoded.

sha_1.update(mfstcontent.encode("utf-8"))

else:

sha_1.update(mfstcontent)

return sha_1.hexdigest()

def validate(self, signatures):

"""Verifies whether the signatures for the contents of

if signatures != self.signatures:

raise apx.BadManifestSignatures(self.fmri)

def store(self, mfst_path):

"""Store the manifest contents to disk."""

t_dir = os.path.dirname(mfst_path)

t_prefix = os.path.basename(mfst_path) + "."

try:

os.makedirs(t_dir, mode=PKG_DIR_MODE)

except EnvironmentError, e:

if e.errno == errno.EACCES:

raise apx.PermissionsException(e.filename)

if e.errno == errno.EROFS:

raise apx.ReadOnlyFileSystemException(

e.filename)

if e.errno != errno.EEXIST:

raise

try:

fd, fn = tempfile.mkstemp(dir=t_dir, prefix=t_prefix)

except EnvironmentError, e:

if e.errno == errno.EACCES:

raise apx.PermissionsException(e.filename)

if e.errno == errno.EROFS:

raise apx.ReadOnlyFileSystemException(

e.filename)

raise

mfile = os.fdopen(fd, "wb")

#

# We specifically avoid sorting manifests before writing

# them to disk-- there's really no point in doing so, since

# we'll sort actions globally during packaging operations.

#

mfile.write(self.tostr_unsorted())

mfile.close()

try:

os.chmod(fn, PKG_FILE_MODE)

portable.rename(fn, mfst_path)

except EnvironmentError, e:

if e.errno == errno.EACCES:

raise apx.PermissionsException(e.filename)

if e.errno == errno.EROFS:

raise apx.ReadOnlyFileSystemException(

e.filename)

raise

def get_variants(self, name):

if name not in self.attributes:

return None

variants = self.attributes[name]

if not isinstance(variants, str):

return variants

return [variants]

def get_all_variants(self):

"""Return a dictionary mapping variant tags to their values."""

return variant.VariantCombinationTemplate(dict((

(name, self.attributes[name])

for name in self.attributes

if name.startswith("variant.")

)))

def get(self, key, default):

try:

return self[key]

except KeyError:

return default

def getbool(self, key, default):

"""Returns the boolean of the value of the attribute 'key'."""

ret = self.get(key, default).lower()

if ret == "true":

return True

elif ret == "false":

return False

else:

raise ValueError(_("Attribute value '%s' not 'true' or "

"'false'" % ret))

def get_size(self, excludes=EmptyI):

"""Returns an integer representing the total size, in bytes, of

size = 0

for a in self.gen_actions(excludes):

size += int(a.attrs.get("pkg.size", "0"))

return size

def __getitem__(self, key):

"""Return the value for the package attribute 'key'."""

return self.attributes[key]

def __setitem__(self, key, value):

"""Set the value for the package attribute 'key' to 'value'."""

self.attributes[key] = value

for a in self.actions:

if a.name == "set" and a.attrs["name"] == key:

a.attrs["value"] = value

return

new_attr = AttributeAction(None, name=key, value=value)

self.actions.append(new_attr)

self.actions_bytype.setdefault("set", []).append(new_attr)

def __contains__(self, key):

return key in self.attributes

null = Manifest()

class FactoredManifest(Manifest):

"""This class serves as a wrapper for the Manifest class for callers

def __init__(self, fmri, cache_root, contents=None, excludes=EmptyI,

pathname=None):

"""Raises KeyError exception if factored manifest is not present

Manifest.__init__(self, fmri)

self.__cache_root = cache_root

self.__pathname = pathname

self.excludes = excludes

self.loaded = False

# Do we have a cached copy?

if not os.path.exists(self.pathname):

if not contents:

raise KeyError, fmri

# we have no cached copy; save one

# don't specify excludes so on-disk copy has

# all variants

self.set_content(content=contents)

self.__finiload()

if self.__storeback():

self.__unload()

elif excludes:

self.exclude_content(excludes)

return

# we have a cached copy of the manifest

mdpath = self.__cache_path("manifest.dircache")

# have we computed the dircache?

if not os.path.exists(mdpath): # we're adding cache

self.excludes = EmptyI # to existing manifest

self.__load()

if self.__storeback():

self.__unload()

elif excludes:

self.excludes = excludes

self.__load()

def __cache_path(self, name):

return os.path.join(self.__cache_root, name)

def __load(self):

"""Load all manifest contents from on-disk copy of manifest"""

self.set_content(excludes=self.excludes, pathname=self.pathname)

self.__finiload()

def __unload(self):

"""Unload manifest; used to reduce peak memory comsumption

self.actions = []

self.actions_bytype = {}

self.variants = {}

self.facets = {}

self.attributes = {}

self.loaded = False

def __finiload(self):

"""Finish loading.... this part of initialization is common

self.loaded = True

def __storeback(self):

""" store the current action set; also create per-type

assert self.loaded

try:

self.store(self.pathname)

self.__storebytype()

return True

except apx.PermissionsException:

# this allows us to try to cache new manifests

# when non-root w/o failures.

return False

def __storebytype(self):

""" create manifest.<typename> files to accelerate partial

assert self.loaded

t_dir = self.__cache_root

# Ensure target cache directory and intermediates exist.

misc.makedirs(t_dir)

# create per-action type cache; use rename to avoid

# corrupt files if ^C'd in the middle

for n in self.actions_bytype.keys():

t_prefix = "manifest.%s." % n

fd, fn = tempfile.mkstemp(dir=t_dir, prefix=t_prefix)

f = os.fdopen(fd, "wb")

for a in self.actions_bytype[n]:

f.write("%s\n" % a)

f.close()

os.chmod(fn, PKG_FILE_MODE)

portable.rename(fn, self.__cache_path("manifest.%s" % n))

# create dircache

fd, fn = tempfile.mkstemp(dir=t_dir,

prefix="manifest.dircache.")

f = os.fdopen(fd, "wb")

dirs = self.__actions_to_dirs()

for s in self.__gen_dirs_to_str(dirs):

f.write(s)

f.close()

os.chmod(fn, PKG_FILE_MODE)

portable.rename(fn, self.__cache_path("manifest.dircache"))

@staticmethod

def __gen_dirs_to_str(dirs):

""" from a dictionary of paths, generate contents of dircache

for d in dirs:

for v in dirs[d]:

yield "dir path=%s %s\n" % \

(d, " ".join("%s=%s" % t \

for t in v.iteritems()))

def __actions_to_dirs(self):

""" create dictionary of all directories referenced

assert self.loaded

dirs = {}

# build a dictionary containing all directories tagged w/

# variants

for a in self.actions:

v, f = a.get_varcet_keys()

variants = dict((name, a.attrs[name]) for name in v + f)

for d in expanddirs(a.directory_references()):

if d not in dirs:

dirs[d] = [variants]

elif variants not in dirs[d]:

dirs[d].append(variants)

# remove any tags if any entries are always installed (NULL)

for d in dirs:

if {} in dirs[d]:

dirs[d] = [{}]

continue

# could collapse dirs where all variants are present

return dirs

@staticmethod

def clear_cache(cache_root):

"""Remove all manifest cache files found in the given directory

try:

for cname in os.listdir(cache_root):

if not cname.startswith("manifest."):

continue

try:

portable.remove(os.path.join(

cache_root, cname))

except EnvironmentError, e:

if e.errno != errno.ENOENT:

raise

except EnvironmentError, e:

apx._convert_error(e, ignored_errors=[errno.ENOENT])

def get_directories(self, excludes):

""" return a list of directories implicitly or

mpath = self.__cache_path("manifest.dircache")

if not os.path.exists(mpath):

# no cached copy

if not self.loaded:

# need to load from disk

self.__load()

# generate actions that contain directories

alist = [

actions.fromstr(s.strip())

for s in self.__gen_dirs_to_str(

self.__actions_to_dirs())

]

else:

# we have cached copy on disk; use it

f = file(mpath)

alist = [actions.fromstr(s.strip()) for s in f]

f.close()

s = set([

a.attrs["path"]

for a in alist

if a.include_this(excludes)

])

return list(s)

def gen_actions_by_type(self, atype, excludes=EmptyI):

""" generate actions of the specified type;

if self.loaded: #if already loaded, use in-memory cached version

# invoke subclass method to generate action by action

for a in Manifest.gen_actions_by_type(self, atype,

excludes):

yield a

return

mpath = self.__cache_path("manifest.dircache")

if not os.path.exists(mpath):

# no cached copy :-(

if not self.loaded:

# get manifest from disk

self.__load()

# invoke subclass method to generate action by action

for a in Manifest.gen_actions_by_type(self, atype,

excludes):

yield a

else:

# we have a cached copy - use it

mpath = self.__cache_path("manifest.%s" % atype)

if not os.path.exists(mpath):

return # no such action in this manifest

f = file(mpath)

for l in f:

a = actions.fromstr(l.strip())

if a.include_this(excludes):

yield a

f.close()

def __load_attributes(self):

"""Load attributes dictionary from cached set actions;

mpath = self.__cache_path("manifest.set")

if not os.path.exists(mpath):

return False

f = file(mpath)

for l in f:

a = actions.fromstr(l.strip())

if a.include_this(self.excludes):

self.fill_attributes(a)

f.close()

return True

def __getitem__(self, key):

if not self.loaded and not self.__load_attributes():

self.__load()

return Manifest.__getitem__(self, key)

def __setitem__(self, key, value):

"""No assignments to factored manifests allowed."""

assert "FactoredManifests are not dicts"

def __contains__(self, key):

if not self.loaded and not self.__load_attributes():

self.__load()

return Manifest.__contains__(self, key)

def get(self, key, default):

try:

return self[key]

except KeyError:

return default

def get_variants(self, name):

if not self.loaded and not self.__load_attributes():

self.__load()

return Manifest.get_variants(self, name)

def get_all_variants(self):

if not self.loaded and not self.__load_attributes():

self.__load()

return Manifest.get_all_variants(self)

@staticmethod

def search_dict(cache_path, excludes, return_line=False):

return Manifest.search_dict(cache_path, excludes,

return_line=return_line)

def gen_actions(self, excludes=EmptyI):

if not self.loaded:

self.__load()

return Manifest.gen_actions(self, excludes=excludes)

def __str__(self, excludes=EmptyI):

if not self.loaded:

self.__load()

return Manifest.__str__(self)

def duplicates(self, excludes=EmptyI):

if not self.loaded:

self.__load()

return Manifest.duplicates(self, excludes=excludes)

def difference(self, origin, origin_exclude=EmptyI,

self_exclude=EmptyI):

if not self.loaded:

self.__load()

return Manifest.difference(self, origin,

origin_exclude=origin_exclude,

self_exclude=self_exclude)

@property

def pathname(self):

"""The absolute pathname of the file containing the manifest."""

if self.__pathname:

return self.__pathname

return os.path.join(self.__cache_root, "manifest")

class EmptyFactoredManifest(Manifest):

"""Special class for pkgplan's need for a empty manifest;

def __init__(self):

Manifest.__init__(self)

def difference(self, origin, origin_exclude=EmptyI,

self_exclude=EmptyI):

"""Return three lists of action pairs representing origin and

# The difference for this case is simply everything in the

# origin has been removed. This is an optimization for

# uninstall.

return ManifestDifference([], [],

[(a, None) for a in origin.gen_actions(origin_exclude)])

@staticmethod

def get_directories(excludes):

return []

NullFactoredManifest = EmptyFactoredManifest()