50N/A

202N/A

50N/Aimport os

50N/Aimport errno

72N/Aimport tempfile

119N/Afrom itertools import groupby, chain, repeat

50N/Afrom pkg.misc import EmptyI, expanddirs

50N/A

50N/Aimport pkg.actions as actions

50N/Aimport pkg.client.api_errors as api_errors

66N/Aimport pkg.portable as portable

135N/Aimport pkg.variant as variant

66N/Aimport sha

66N/A

50N/Afrom pkg.actions.attribute import AttributeAction

235N/A

50N/Aclass Manifest(object):

50N/A """A Manifest is the representation of the actions composing a specific

66N/A

242N/A def __init__(self):

66N/A self.img = None

195N/A self.fmri = None

66N/A

66N/A self.actions = []

50N/A self.actions_bytype = {}

235N/A self.variants = {} # variants seen in package

235N/A self.facets = {} # facets seen in package

235N/A self.attributes = {} # package-wide attributes

235N/A

235N/A def __str__(self):

50N/A r = ""

111N/A if "fmri" not in self.attributes and self.fmri != None:

66N/A r += "set name=fmri value=%s\n" % self.fmri

50N/A

50N/A for act in sorted(self.actions):

66N/A r += "%s\n" % act

66N/A return r

66N/A

50N/A def as_lines(self):

72N/A """A generator function that returns the unsorted manifest

258N/A

258N/A if "fmri" not in self.attributes and self.fmri != None:

59N/A yield "set name=fmri value=%s\n" % self.fmri

72N/A

50N/A for act in self.actions:

111N/A yield "%s\n" % act

111N/A

111N/A def tostr_unsorted(self):

111N/A return "".join((l for l in self.as_lines()))

111N/A

111N/A def difference(self, origin, origin_exclude=EmptyI,

111N/A self_exclude=EmptyI):

111N/A """Return three lists of action pairs representing origin and

111N/A # XXX Do we need to find some way to assert that the keys are

111N/A # all unique?

111N/A

111N/A if isinstance(origin, EmptyCachedManifest):

111N/A # No origin was provided, so nothing has been changed or

111N/A # removed; only added. In addition, this doesn't need

111N/A # to be sorted since the caller likely already does

111N/A # (such as pkgplan/imageplan).

111N/A return (

111N/A [(None, a) for a in self.gen_actions(self_exclude)],

111N/A [], [])

111N/A

111N/A sdict = dict(

111N/A ((a.name, a.attrs.get(a.key_attr, id(a))), a)

111N/A for a in self.gen_actions(self_exclude)

111N/A )

72N/A odict = dict(

72N/A ((a.name, a.attrs.get(a.key_attr, id(a))), a)

50N/A for a in origin.gen_actions(origin_exclude)

237N/A )

237N/A

237N/A sset = set(sdict.keys())

237N/A oset = set(odict.keys())

237N/A

237N/A added = [(None, sdict[i]) for i in sset - oset]

237N/A removed = [(odict[i], None) for i in oset - sset]

237N/A # XXX for now, we force license actions to always be

237N/A # different to insure that existing license files for

237N/A # new versions are always installed

237N/A changed = [

237N/A (odict[i], sdict[i])

235N/A for i in oset & sset

235N/A if odict[i].different(sdict[i]) or i[0] == "license"

235N/A ]

235N/A

235N/A # XXX Do changed actions need to be sorted at all? This is

235N/A # likely to be the largest list, so we might save significant

235N/A # time by not sorting. Should we sort above? Insert into a

235N/A # sorted list?

235N/A

235N/A # singlesort = lambda x: x[0] or x[1]

235N/A addsort = lambda x: x[1]

235N/A remsort = lambda x: x[0]

235N/A removed.sort(key = remsort, reverse = True)

235N/A added.sort(key = addsort)

235N/A changed.sort(key = addsort)

235N/A

235N/A return (added, changed, removed)

235N/A

235N/A @staticmethod

235N/A def comm(*compare_m):

235N/A """Like the unix utility comm, except that this function

235N/A

235N/A # construct list of dictionaries of actions in each

237N/A # manifest, indexed by unique keys

50N/A m_dicts = [

66N/A dict(

135N/A ((a.name, a.attrs.get(a.key_attr, id(a))), a)

66N/A for a in m.actions)

66N/A for m in compare_m

66N/A ]

66N/A # construct list of key sets in each dict

119N/A #

119N/A m_sets = [

119N/A set(m.keys())

235N/A for m in m_dicts

235N/A ]

235N/A

50N/A common_keys = reduce(lambda a, b: a & b, m_sets)

66N/A

195N/A # determine which common_keys have common actions

59N/A for k in common_keys.copy():

135N/A for i in range(len(m_dicts) - 1):

135N/A if m_dicts[i][k].different(

135N/A m_dicts[i + 1][k]):

135N/A common_keys.remove(k)

135N/A break

135N/A return tuple(

135N/A [

111N/A [m_dicts[i][k] for k in m_sets[i] - common_keys]

202N/A for i in range(len(m_dicts))

72N/A ]

136N/A +

66N/A [

136N/A [ m_dicts[0][k] for k in common_keys ]

66N/A ]

119N/A )

119N/A

119N/A

119N/A def combined_difference(self, origin, ov=EmptyI, sv=EmptyI):

119N/A """Where difference() returns three lists, combined_difference()

119N/A return list(chain(*self.difference(origin, ov, sv)))

119N/A

119N/A def humanized_differences(self, other, ov=EmptyI, sv=EmptyI):

119N/A """Output expects that self is newer than other. Use of sets

119N/A

119N/A l = self.difference(other, ov, sv)

119N/A out = ""

119N/A

119N/A for src, dest in chain(*l):

119N/A if not src:

119N/A out += "+ %s\n" % str(dest)

119N/A elif not dest:

119N/A out += "- %s\n" + str(src)

119N/A else:

119N/A out += "%s -> %s\n" % (src, dest)

119N/A return out

235N/A

119N/A def gen_actions(self, excludes=EmptyI):

119N/A """Generate actions in manifest through ordered callable list"""

119N/A for a in self.actions:

119N/A for c in excludes:

235N/A if not c(a):

235N/A break

235N/A else:

237N/A yield a

202N/A

237N/A def gen_actions_by_type(self, atype, excludes=EmptyI):

237N/A """Generate actions in the manifest of type "type"

237N/A for a in self.actions_bytype.get(atype, []):

237N/A for c in excludes:

237N/A if not c(a):

237N/A break

202N/A else:

237N/A yield a

237N/A

237N/A def gen_key_attribute_value_by_type(self, atype, excludes=EmptyI):

237N/A """Generate the value of the key atrribute for each action

237N/A

237N/A return (

237N/A a.attrs.get(a.key_attr)

237N/A for a in self.gen_actions_by_type(atype, excludes)

237N/A )

237N/A

237N/A def duplicates(self, excludes=EmptyI):

202N/A """Find actions in the manifest which are duplicates (i.e.,

237N/A

237N/A def fun(a):

237N/A """Return a key on which actions can be sorted."""

237N/A return a.name, a.attrs.get(a.key_attr, id(a))

237N/A

237N/A alldups = []

237N/A acts = [a for a in self.gen_actions(excludes)]

237N/A

59N/A for k, g in groupby(sorted(acts, key=fun), fun):

237N/A glist = list(g)

237N/A dups = set()

237N/A for i in range(len(glist) - 1):

237N/A if glist[i].different(glist[i + 1]):

237N/A dups.add(glist[i])

237N/A dups.add(glist[i + 1])

237N/A if dups:

237N/A alldups.append((k, dups))

237N/A return alldups

237N/A

237N/A def set_fmri(self, img, fmri):

66N/A self.img = img

66N/A self.fmri = fmri

135N/A

66N/A def set_content(self, content, excludes=EmptyI):

66N/A """content is the text representation of the manifest"""

66N/A self.actions = []

66N/A self.actions_bytype = {}

66N/A self.variants = {}

202N/A self.facets = {}

72N/A self.attributes = {}

136N/A

66N/A # So we could build up here the type/key_attr dictionaries like

136N/A # sdict and odict in difference() above, and have that be our

66N/A # main datastore, rather than the simple list we have now. If

111N/A # we do that here, we can even assert that the "same" action

111N/A # can't be in a manifest twice. (The problem of having the same

66N/A # action more than once in packages that can be installed

66N/A # together has to be solved somewhere else, though.)

195N/A for l in content.splitlines():

66N/A l = l.lstrip()

135N/A if not l or l[0] == "#":

135N/A continue

135N/A

135N/A try:

135N/A action = actions.fromstr(l)

135N/A except actions.ActionError, e:

135N/A # Add the FMRI to the exception and re-raise

111N/A e.fmri = self.fmri

66N/A raise

195N/A # XXX handle legacy transition issues; not needed after

60N/A # 2009.06 release & republication are complete.

111N/A if "opensolaris.zone" in action.attrs and \

111N/A "variant.opensolaris.zone" not in action.attrs:

111N/A action.attrs["variant.opensolaris.zone"] = \

111N/A action.attrs["opensolaris.zone"]

111N/A

111N/A if action.name == "set" and \

111N/A action.attrs["name"] == "authority":

111N/A # Translate old action to new.

242N/A action.attrs["name"] = "publisher"

242N/A

111N/A if action.attrs.has_key("path"):

111N/A np = action.attrs["path"].lstrip(os.path.sep)

235N/A action.attrs["path"] = np

if not action.include_this(excludes):

continue

self.actions.append(action)

if action.name not in self.actions_bytype:

self.actions_bytype[action.name] = [ action ]

else:

self.actions_bytype[action.name].append(action)

# add any set actions to attributes

if action.name == "set":

self.fill_attributes(action)

# append any variants and facets to manifest dict

v_list, f_list = action.get_varcet_keys()

if v_list or f_list:

for v, d in zip(v_list, repeat(self.variants)) \

+ zip(f_list, repeat(self.facets)):

if v not in d:

d[v] = set([action.attrs[v]])

else:

d[v].add(action.attrs[v])

return

def fill_attributes(self, action):

"""Fill attribute array w/ set action contents."""

try:

keyvalue = action.attrs["name"]

if keyvalue not in self.attributes:

self.attributes[keyvalue] = \

action.attrs["value"]

except KeyError: # ignore broken set actions

pass

@staticmethod

def search_dict(file_path, excludes, return_line=False,

log=None):

"""Produces the search dictionary for a specific manifest.

if log is None:

log = lambda x: None

file_handle = file(file_path, "rb")

cur_pos = 0

line = file_handle.readline()

action_dict = {}

def __handle_list(lst, cp):

"""Translates what actions.generate_indices produces

for action_name, subtype, tok, full_value in lst:

if action_name == "set":

if full_value is None:

full_value = tok

else:

if full_value is None:

full_value = subtype

if full_value is None:

full_value = action_name

if isinstance(tok, list):

__handle_list([

(action_name, subtype, t,

full_value)

for t in tok

], cp)

else:

if (tok, action_name, subtype,

full_value) in action_dict:

action_dict[(tok, action_name,

subtype, full_value)

].append(cp)

else:

action_dict[(tok, action_name,

subtype, full_value)] = [cp]

while line:

l = line.strip()

if l and l[0] != "#":

try:

action = actions.fromstr(l)

except actions.ActionError, e:

log((_("%(fp)s:\n%(e)s") %

{ "fp": file_path, "e": e }))

else:

if action.include_this(excludes):

if action.attrs.has_key("path"):

np = action.attrs["path"].lstrip(os.path.sep)

action.attrs["path"] = \

np

try:

inds = action.generate_indices()

except KeyError, k:

log(_("%(fp)s contains "

"an action which is"

" missing the "

"expected attribute"

": %(at)s.\nThe "

"action is:"

"%(act)s") %

{

"fp": file_path,

"at": k.args[0],

"act":l

})

else:

arg = cur_pos

if return_line:

arg = l

__handle_list(inds, arg)

cur_pos = file_handle.tell()

line = file_handle.readline()

file_handle.close()

return action_dict

@property

def signatures(self):

"""A dict structure of signature key and value pairs for the

# NOTE: The logic here must be identical to that of store() so

# that the signature properly reflects the on-disk data.

sha_1 = sha.new()

for l in self.as_lines():

sha_1.update(l)

return { "sha-1": sha_1.hexdigest() }

def validate(self, signatures):

"""Verifies whether the signatures for the contents of

if signatures != self.signatures:

raise api_errors.BadManifestSignatures(self.fmri)

def store(self, mfst_path):

"""Store the manifest contents to disk."""

t_dir = os.path.dirname(mfst_path)

t_prefix = os.path.basename(mfst_path) + "."

if not os.path.exists(t_dir):

os.makedirs(t_dir)

fd, fn = tempfile.mkstemp(dir=t_dir, prefix=t_prefix)

mfile = os.fdopen(fd, "wb")

#

# We specifically avoid sorting manifests before writing

# them to disk-- there's really no point in doing so, since

# we'll sort actions globally during packaging operations.

#

mfile.write(self.tostr_unsorted())

mfile.close()

os.chmod(fn, 0644)

portable.rename(fn, mfst_path)

def get_variants(self, name):

if name not in self.attributes:

return None

variants = self.attributes[name]

if not isinstance(variants, str):

return variants

return [variants]

def get_all_variants(self):

"""Return a dictionary mapping variant tags to their values."""

return variant.VariantSets(dict((

(name, self.attributes[name])

for name in self.attributes

if name.startswith("variant.")

)))

def get(self, key, default):

try:

return self[key]

except KeyError:

return default

def get_size(self, excludes=EmptyI):

"""Returns an integer representing the total size, in bytes, of

size = 0

for a in self.gen_actions(excludes):

size += int(a.attrs.get("pkg.size", "0"))

return size

def __getitem__(self, key):

"""Return the value for the package attribute 'key'."""

return self.attributes[key]

def __setitem__(self, key, value):

"""Set the value for the package attribute 'key' to 'value'."""

self.attributes[key] = value

for a in self.actions:

if a.name == "set" and a.attrs["name"] == key:

a.attrs["value"] = value

return

new_attr = AttributeAction(None, name=key, value=value)

self.actions.append(new_attr)

def __contains__(self, key):

return key in self.attributes

null = Manifest()

class CachedManifest(Manifest):

"""This class handles a cache of manifests for the client;

def __file_path(self, name):

return os.path.join(self.__file_dir(), name)

def __file_dir(self):

return os.path.join(self.__pkgdir,

self.fmri.get_dir_path())

def __init__(self, fmri, pkgdir, preferred_pub, excludes=EmptyI,

contents=None):

"""Raises KeyError exception if cached manifest

Manifest.__init__(self)

self.__pkgdir = pkgdir

self.__pub = preferred_pub

self.loaded = False

self.set_fmri(None, fmri)

self.excludes = excludes

mpath = self.__file_path("manifest")

# Do we have a cached copy?

if not os.path.exists(mpath):

if not contents:

raise KeyError, fmri

# we have no cached copy; save one

# don't specify excludes so on-disk copy has

# all variants

self.set_content(contents)

self.__finiload()

if self.__storeback():

self.__unload()

elif excludes:

self.set_content(contents, excludes)

return

# we have a cached copy of the manifest

mdpath = self.__file_path("manifest.dircache")

# have we computed the dircache?

if not os.path.exists(mdpath): # we're adding cache

self.excludes = EmptyI # to existing manifest

self.__load()

if self.__storeback():

self.__unload()

elif excludes:

self.excludes = excludes

self.__load()

def __load(self):

"""Load all manifest contents from on-disk copy of manifest"""

f = file(self.__file_path("manifest"))

data = f.read()

f.close()

self.set_content(data, self.excludes)

self.__finiload()

def __unload(self):

"""Unload manifest; used to reduce peak memory comsumption

self.actions = []

self.actions_bytype = {}

self.variants = {}

self.facets = {}

self.attributes = {}

self.loaded = False

def __finiload(self):

"""Finish loading.... this part of initialization is common

self.loaded = True

# this needs to change; we should not modify on-disk manifest

if self.fmri and "publisher" not in self.attributes:

if not self.fmri.has_publisher():

pub = self.__pub

else:

pub = self.fmri.get_publisher()

# This shouldn't be set unless available.

if pub:

Manifest.__setitem__(self, "publisher", pub)

def __storeback(self):

""" store the current action set; also create per-type

assert self.loaded

try:

self.store(self.__file_path("manifest"))

self.__storebytype()

return True

except EnvironmentError, e:

# this allows us to try to cache new manifests

# when non-root w/o failures

if e.errno not in (errno.EROFS, errno.EACCES):

raise

return False

def __storebytype(self):

""" create manifest.<typename> files to accelerate partial

assert self.loaded

t_dir = self.__file_dir()

# create per-action type cache; use rename to avoid

# corrupt files if ^C'd in the middle

for n in self.actions_bytype.keys():

t_prefix = "manifest.%s." % n

fd, fn = tempfile.mkstemp(dir=t_dir, prefix=t_prefix)

f = os.fdopen(fd, "wb")

for a in self.actions_bytype[n]:

f.write("%s\n" % a)

f.close()

os.chmod(fn, 0644)

portable.rename(fn, self.__file_path("manifest.%s" % n))

# create dircache

fd, fn = tempfile.mkstemp(dir=t_dir,

prefix="manifest.dircache.")

f = os.fdopen(fd, "wb")

dirs = self.__actions_to_dirs()

for s in self.__gen_dirs_to_str(dirs):

f.write(s)

f.close()

os.chmod(fn, 0644)

portable.rename(fn, self.__file_path("manifest.dircache"))

@staticmethod

def __gen_dirs_to_str(dirs):

""" from a dictionary of paths, generate contents of dircache

for d in dirs:

for v in dirs[d]:

yield "dir path=%s %s\n" % \

(d, " ".join("%s=%s" % t \

for t in v.iteritems()))

def __actions_to_dirs(self):

""" create dictionary of all directories referenced

assert self.loaded

dirs = {}

# build a dictionary containing all directories tagged w/

# variants

for a in self.actions:

v, f = a.get_varcet_keys()

variants = dict((name, a.attrs[name]) for name in v + f)

for d in expanddirs(a.directory_references()):

if d not in dirs:

dirs[d] = [variants]

elif variants not in dirs[d]:

dirs[d].append(variants)

# remove any tags if any entries are always installed (NULL)

for d in dirs:

if {} in dirs[d]:

dirs[d] = [{}]

continue

# could collapse dirs where all variants are present

return dirs

def get_directories(self, excludes):

""" return a list of directories implicitly or

mpath = self.__file_path("manifest.dircache")

if not os.path.exists(mpath):

# no cached copy

if not self.loaded:

# need to load from disk

self.__load()

# generate actions that contain directories

alist = [

actions.fromstr(s.strip())

for s in self.__gen_dirs_to_str(

self.__actions_to_dirs())

]

else:

# we have cached copy on disk; use it

f = file(mpath)

alist = [actions.fromstr(s.strip()) for s in f]

f.close()

s = set([

a.attrs["path"]

for a in alist

if a.include_this(excludes)

])

return list(s)

def gen_actions_by_type(self, atype, excludes=EmptyI):

""" generate actions of the specified type;

if self.loaded: #if already loaded, use in-memory cached version

# invoke subclass method to generate action by action

for a in Manifest.gen_actions_by_type(self, atype,

excludes):

yield a

return

mpath = self.__file_path("manifest.dircache")

if not os.path.exists(mpath):

# no cached copy :-(

if not self.loaded:

# get manifest from disk

self.__load()

# invoke subclass method to generate action by action

for a in Manifest.gen_actions_by_type(self, atype,

excludes):

yield a

else:

# we have a cached copy - use it

mpath = self.__file_path("manifest.%s" % atype)

if not os.path.exists(mpath):

return # no such action in this manifest

f = file(mpath)

for l in f:

a = actions.fromstr(l.strip())

if a.include_this(excludes):

yield a

f.close()

def __load_attributes(self):

"""Load attributes dictionary from cached set actions;

mpath = self.__file_path("manifest.set")

if not os.path.exists(mpath):

return False

f = file(mpath)

for l in f:

a = actions.fromstr(l.strip())

if a.include_this(self.excludes):

self.fill_attributes(a)

f.close()

return True

def __getitem__(self, key):

if not self.loaded and not self.__load_attributes():

self.__load()

return Manifest.__getitem__(self, key)

def __setitem__(self, key, value):

"""No assignments to cached manifests allowed."""

assert "CachedManifests are not dicts"

def __contains__(self, key):

if not self.loaded and not self.__load_attributes():

self.__load()

return Manifest.__contains__(self, key)

def get(self, key, default):

try:

return self[key]

except KeyError:

return default

def get_variants(self, name):

if not self.loaded and not self.__load_attributes():

self.__load()

return Manifest.get_variants(self, name)

def get_all_variants(self):

if not self.loaded and not self.__load_attributes():

self.__load()

return Manifest.get_all_variants(self)

@staticmethod

def search_dict(file_path, excludes, return_line=False):

return Manifest.search_dict(file_path, excludes,

return_line=return_line)

def gen_actions(self, excludes=EmptyI):

if not self.loaded:

self.__load()

return Manifest.gen_actions(self, excludes=excludes)

def duplicates(self, excludes=EmptyI):

if not self.loaded:

self.__load()

return Manifest.duplicates(self, excludes=excludes)

def difference(self, origin, origin_exclude=EmptyI,

self_exclude=EmptyI):

if not self.loaded:

self.__load()

return Manifest.difference(self, origin,

origin_exclude=origin_exclude,

self_exclude=self_exclude)

class EmptyCachedManifest(Manifest):

"""Special class for pkgplan's need for a empty manifest;

def __init__(self):

Manifest.__init__(self)

def difference(self, origin, origin_exclude=EmptyI,

self_exclude=EmptyI):

"""Return three lists of action pairs representing origin and

# The difference for this case is simply everything in the

# origin has been removed. This is an optimization for

# uninstall.

return ([], [],

[(a, None) for a in origin.gen_actions(self_exclude)])

@staticmethod

def get_directories(excludes):

return []

NullCachedManifest = EmptyCachedManifest()