manifest.py revision 956
409N/A#!/usr/bin/python2.4
39N/A#
39N/A# CDDL HEADER START
39N/A#
39N/A# The contents of this file are subject to the terms of the
39N/A# Common Development and Distribution License (the "License").
39N/A# You may not use this file except in compliance with the License.
39N/A#
39N/A# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
39N/A# or http://www.opensolaris.org/os/licensing.
39N/A# See the License for the specific language governing permissions
39N/A# and limitations under the License.
39N/A#
39N/A# When distributing Covered Code, include this CDDL HEADER in each
39N/A# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
39N/A# If applicable, add the following below this CDDL HEADER, with the
39N/A# fields enclosed by brackets "[]" replaced with your own identifying
39N/A# information: Portions Copyright [yyyy] [name of copyright owner]
39N/A#
39N/A# CDDL HEADER END
39N/A#
227N/A
39N/A#
39N/A# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
39N/A# Use is subject to license terms.
342N/A#
203N/A
39N/Aimport os
51N/Aimport errno
111N/Afrom itertools import groupby, chain, repeat
296N/Afrom pkg.misc import EmptyI, expanddirs
39N/A
39N/Aimport pkg.actions as actions
39N/Afrom pkg.actions.attribute import AttributeAction
39N/A
39N/Aclass Manifest(object):
39N/A """A Manifest is the representation of the actions composing a specific
39N/A package version on both the client and the repository. Both purposes
39N/A utilize the same storage format.
39N/A
39N/A The serialized structure of a manifest is an unordered list of actions.
39N/A
48N/A The special action, "set", represents a package attribute.
48N/A
48N/A The reserved attribute, "fmri", represents the package and version
48N/A described by this manifest. It is available as a string via the
429N/A attributes dictionary, and as an FMRI object from the fmri member.
48N/A
48N/A The list of manifest-wide reserved attributes is
48N/A
48N/A base_directory Default base directory, for non-user images.
48N/A fmri Package FMRI.
39N/A isa Package is intended for a list of ISAs.
39N/A platform Package is intended for a list of platforms.
39N/A relocatable Suitable for User Image.
39N/A
39N/A All non-prefixed attributes are reserved to the framework. Third
39N/A parties may prefix their attributes with a reversed domain name, domain
205N/A name, or stock symbol. An example might be
39N/A
205N/A com.example,supported
39N/A
39N/A as an indicator that a specific package version is supported by the
39N/A vendor, example.com.
39N/A
39N/A manifest.null is provided as the null manifest. Differences against the
39N/A null manifest result in the complete set of attributes and actions of
39N/A the non-null manifest, meaning that all operations can be viewed as
39N/A tranitions between the manifest being installed and the manifest already
39N/A present in the image (which may be the null manifest).
39N/A """
39N/A
39N/A def __init__(self):
39N/A self.img = None
39N/A self.fmri = None
39N/A
39N/A self.size = 0
39N/A self.actions = []
39N/A self.actions_bytype = {}
39N/A self.variants = {} # variants seen in package
39N/A self.facets = {} # facets seen in package
39N/A self.attributes = {} # package-wide attributes
39N/A
48N/A def __str__(self):
48N/A r = ""
48N/A if "fmri" not in self.attributes and self.fmri != None:
48N/A r += "set name=fmri value=%s\n" % self.fmri
48N/A
59N/A for act in sorted(self.actions):
48N/A r += "%s\n" % act
39N/A return r
104N/A
39N/A def tostr_unsorted(self):
39N/A r = ""
301N/A if "fmri" not in self.attributes and self.fmri != None:
39N/A r += "set name=fmri value=%s\n" % self.fmri
567N/A
39N/A for act in self.actions:
39N/A r += "%s\n" % act
39N/A return r
39N/A
315N/A
39N/A def difference(self, origin, origin_exclude=EmptyI,
227N/A self_exclude=EmptyI):
315N/A """Return three lists of action pairs representing origin and
315N/A destination actions. The first list contains the pairs
39N/A representing additions, the second list contains the pairs
315N/A representing updates, and the third list contains the pairs
315N/A represnting removals. All three lists are in the order in which
315N/A they should be executed."""
315N/A # XXX Do we need to find some way to assert that the keys are
429N/A # all unique?
315N/A
315N/A sdict = dict(
39N/A ((a.name, a.attrs.get(a.key_attr, id(a))), a)
429N/A for a in self.gen_actions(self_exclude)
39N/A )
72N/A odict = dict(
203N/A ((a.name, a.attrs.get(a.key_attr, id(a))), a)
203N/A for a in origin.gen_actions(origin_exclude)
203N/A )
203N/A
203N/A sset = set(sdict.keys())
203N/A oset = set(odict.keys())
72N/A
72N/A added = [(None, sdict[i]) for i in sset - oset]
59N/A removed = [(odict[i], None) for i in oset - sset]
72N/A # XXX for now, we force license actions to always be
72N/A # different to insure that existing license files for
72N/A # new versions are always installed
72N/A changed = [
72N/A (odict[i], sdict[i])
72N/A for i in oset & sset
72N/A if odict[i].different(sdict[i]) or i[0] == "license"
72N/A ]
59N/A
72N/A # XXX Do changed actions need to be sorted at all? This is
72N/A # likely to be the largest list, so we might save significant
59N/A # time by not sorting. Should we sort above? Insert into a
72N/A # sorted list?
72N/A
307N/A # singlesort = lambda x: x[0] or x[1]
307N/A addsort = lambda x: x[1]
307N/A remsort = lambda x: x[0]
72N/A removed.sort(key = remsort, reverse = True)
72N/A added.sort(key = addsort)
72N/A changed.sort(key = addsort)
237N/A
72N/A return (added, changed, removed)
59N/A
72N/A @staticmethod
72N/A def comm(*compare_m):
72N/A """Like the unix utility comm, except that this function
72N/A takes an arbitrary number of manifests and compares them,
59N/A returning a tuple consisting of each manifest's actions
72N/A that are not the same for all manifests, followed by a
72N/A list of actions that are the same in each manifest."""
72N/A
202N/A # construct list of dictionaries of actions in each
72N/A # manifest, indexed by unique keys
72N/A m_dicts = [
59N/A dict(
202N/A ((a.name, a.attrs.get(a.key_attr, id(a))), a)
59N/A for a in m.actions)
203N/A for m in compare_m
203N/A ]
315N/A # construct list of key sets in each dict
203N/A #
203N/A m_sets = [
181N/A set(m.keys())
48N/A for m in m_dicts
48N/A ]
48N/A
48N/A common_keys = reduce(lambda a, b: a & b, m_sets)
48N/A
72N/A # determine which common_keys have common actions
203N/A for k in common_keys.copy():
48N/A for i in range(len(m_dicts) - 1):
203N/A if m_dicts[i][k].different(
72N/A m_dicts[i + 1][k]):
181N/A common_keys.remove(k)
72N/A break
181N/A return tuple(
46N/A [
181N/A [m_dicts[i][k] for k in m_sets[i] - common_keys]
237N/A for i in range(len(m_dicts))
46N/A ]
342N/A +
342N/A [
342N/A [ m_dicts[0][k] for k in common_keys ]
567N/A ]
307N/A )
111N/A
111N/A
111N/A def combined_difference(self, origin, ov=EmptyI, sv=EmptyI):
111N/A """Where difference() returns three lists, combined_difference()
111N/A returns a single list of the concatenation of the three."""
111N/A return list(chain(*self.difference(origin, ov, sv)))
111N/A
111N/A def humanized_differences(self, other, ov=EmptyI, sv=EmptyI):
111N/A """Output expects that self is newer than other. Use of sets
111N/A requires that we convert the action objects into some marshalled
111N/A form, otherwise set member identities are derived from the
111N/A object pointers, rather than the contents."""
111N/A
111N/A l = self.difference(other, ov, sv)
111N/A out = ""
111N/A
111N/A for src, dest in chain(*l):
111N/A if not src:
113N/A out += "+ %s\n" % str(dest)
111N/A elif not dest:
113N/A out += "- %s\n" + str(src)
113N/A else:
113N/A out += "%s -> %s\n" % (src, dest)
113N/A return out
113N/A
113N/A def gen_actions(self, excludes=EmptyI):
113N/A """Generate actions in manifest through ordered callable list"""
113N/A for a in self.actions:
113N/A for c in excludes:
111N/A if not c(a):
104N/A break
104N/A else:
39N/A yield a
39N/A
39N/A def gen_actions_by_type(self, atype, excludes=EmptyI):
39N/A """Generate actions in the manifest of type "type"
307N/A through ordered callable list"""
301N/A for a in self.actions_bytype.get(atype, []):
72N/A for c in excludes:
72N/A if not c(a):
72N/A break
72N/A else:
72N/A yield a
72N/A
72N/A def gen_key_attribute_value_by_type(self, atype, excludes=EmptyI):
39N/A """Generate the value of the key atrribute for each action
146N/A of type "type" in the manifest."""
146N/A
50N/A return (
51N/A a.attrs.get(a.key_attr)
51N/A for a in self.gen_actions_by_type(atype, excludes)
51N/A )
315N/A
315N/A def duplicates(self, excludes=EmptyI):
51N/A """Find actions in the manifest which are duplicates (i.e.,
123N/A represent the same object) but which are not identical (i.e.,
123N/A have all the same attributes)."""
123N/A
123N/A def fun(a):
307N/A """Return a key on which actions can be sorted."""
429N/A return a.name, a.attrs.get(a.key_attr, id(a))
39N/A
567N/A alldups = []
567N/A acts = [a for a in self.gen_actions(excludes)]
567N/A
567N/A for k, g in groupby(sorted(acts, key=fun), fun):
567N/A glist = list(g)
39N/A dups = set()
39N/A for i in range(len(glist) - 1):
144N/A if glist[i].different(glist[i + 1]):
144N/A dups.add(glist[i])
144N/A dups.add(glist[i + 1])
144N/A if dups:
144N/A alldups.append((k, dups))
508N/A return alldups
508N/A
508N/A def set_fmri(self, img, fmri):
508N/A self.img = img
508N/A self.fmri = fmri
429N/A
429N/A def set_content(self, content, excludes=EmptyI):
429N/A """content is the text representation of the manifest"""
429N/A self.size = 0
429N/A self.actions = []
429N/A self.actions_bytype = {}
429N/A self.variants = {}
429N/A self.facets = {}
429N/A self.attributes = {}
429N/A
429N/A # So we could build up here the type/key_attr dictionaries like
429N/A # sdict and odict in difference() above, and have that be our
429N/A # main datastore, rather than the simple list we have now. If
429N/A # we do that here, we can even assert that the "same" action
429N/A # can't be in a manifest twice. (The problem of having the same
429N/A # action more than once in packages that can be installed
429N/A # together has to be solved somewhere else, though.)
429N/A for l in content.splitlines():
429N/A l = l.lstrip()
429N/A if not l or l[0] == "#":
144N/A continue
429N/A
429N/A try:
429N/A action = actions.fromstr(l)
429N/A except actions.ActionError, e:
429N/A # Add the FMRI to the exception and re-raise
429N/A e.fmri = self.fmri
302N/A raise
429N/A
429N/A if action.name == "set" and \
429N/A action.attrs["name"] == "authority":
429N/A # Translate old action to new.
429N/A action.attrs["name"] = "publisher"
144N/A
429N/A if action.attrs.has_key("path"):
429N/A np = action.attrs["path"].lstrip(os.path.sep)
144N/A action.attrs["path"] = np
144N/A
429N/A if not action.include_this(excludes):
429N/A continue
144N/A
315N/A self.size += int(action.attrs.get("pkg.size", "0"))
315N/A self.actions.append(action)
315N/A
342N/A if action.name not in self.actions_bytype:
342N/A self.actions_bytype[action.name] = [ action ]
342N/A else:
342N/A self.actions_bytype[action.name].append(action)
342N/A # add any set actions to attributes
315N/A if action.name == "set":
315N/A self.fill_attributes(action)
315N/A # append any variants and facets to manifest dict
315N/A v_list, f_list = action.get_varcet_keys()
315N/A
315N/A if v_list or f_list:
315N/A for v, d in zip(v_list, repeat(self.variants)) \
315N/A + zip(f_list, repeat(self.facets)):
315N/A if v not in d:
144N/A d[v] = set([action.attrs[v]])
205N/A else:
257N/A d[v].add(action.attrs[v])
257N/A return
257N/A
205N/A def fill_attributes(self, action):
205N/A """Fill attribute array w/ set action contents."""
205N/A try:
257N/A keyvalue = action.attrs["name"]
257N/A if keyvalue not in self.attributes:
257N/A self.attributes[keyvalue] = \
257N/A action.attrs["value"]
257N/A except KeyError: # ignore broken set actions
257N/A pass
257N/A
257N/A
257N/A @staticmethod
257N/A def search_dict(file_path, excludes, return_line=False):
257N/A file_handle = file(file_path)
257N/A cur_pos = 0
257N/A line = file_handle.readline()
257N/A action_dict = {}
205N/A def __handle_list(lst, cp):
205N/A for action_name, subtype, tok, full_value in lst:
205N/A if action_name == "set":
257N/A if full_value is None:
257N/A full_value = tok
205N/A else:
296N/A if full_value is None:
296N/A full_value = subtype
296N/A if full_value is None:
296N/A full_value = action_name
296N/A if isinstance(tok, list):
296N/A __handle_list([
296N/A (action_name, subtype, t,
296N/A full_value)
296N/A for t in tok
296N/A ], cp)
205N/A else:
205N/A if (tok, action_name, subtype,
205N/A full_value) in action_dict:
205N/A action_dict[(tok, action_name,
205N/A subtype, full_value)
205N/A ].append(cp)
48N/A else:
action_dict[(tok, action_name,
subtype, full_value)] = [cp]
while line:
l = line.strip()
if l and l[0] != "#":
action = actions.fromstr(l)
if action.include_this(excludes):
if action.attrs.has_key("path"):
np = action.attrs["path"].lstrip(os.path.sep)
action.attrs["path"] = np
arg = cur_pos
if return_line:
arg = l
__handle_list(action.generate_indices(),
arg)
cur_pos = file_handle.tell()
line = file_handle.readline()
file_handle.close()
return action_dict
def store(self, mfst_path):
"""Store the manifest contents to disk."""
try:
mfile = file(mfst_path, "w")
except IOError:
try:
os.makedirs(os.path.dirname(mfst_path))
except OSError, e:
if e.errno != errno.EEXIST:
raise
mfile = file(mfst_path, "w")
#
# We specifically avoid sorting manifests before writing
# them to disk-- there's really no point in doing so, since
# we'll sort actions globally during packaging operations.
#
mfile.write(self.tostr_unsorted())
mfile.close()
def get_variants(self, name):
if name not in self.attributes:
return None
variants = self.attributes[name]
if not isinstance(variants, str):
return variants
return [variants]
def get(self, key, default):
try:
return self[key]
except KeyError:
return default
def __getitem__(self, key):
"""Return the value for the package attribute 'key'."""
return self.attributes[key]
def __setitem__(self, key, value):
"""Set the value for the package attribute 'key' to 'value'."""
self.attributes[key] = value
for a in self.actions:
if a.name == "set" and a.attrs["name"] == key:
a.attrs["value"] = value
return
new_attr = AttributeAction(None, name=key, value=value)
self.actions.append(new_attr)
def __contains__(self, key):
return key in self.attributes
null = Manifest()
class CachedManifest(Manifest):
"""This class handles a cache of manifests for the client;
it partitions the manifest into multiple files (one per
action type) and also builds an on-disk cache of the
directories explictly and implicitly referenced by the
manifest, tagging each one w/ the appropriate variants/facets."""
__pkgdir = None
__pub = None
@staticmethod
def initialize(pkgdir, default_publisher):
"""get location for manifests, default publisher
of this image"""
CachedManifest.__pkgdir = pkgdir
CachedManifest.__pub = default_publisher
def __file_path(self, file):
return os.path.join(self.__pkgdir,
self.fmri.get_dir_path(), file)
def __init__(self, fmri, excludes=EmptyI, contents=None):
"""Raises KeyError exception if cached manifest
is not present and contents are None; delays
reading of manifest until required if cache file
is present"""
Manifest.__init__(self)
assert self.__pub
self.loaded = False
self.set_fmri(None, fmri)
self.excludes = excludes
mpath = self.__file_path("manifest")
# Do we have a cached copy?
if not os.path.exists(mpath):
if not contents:
raise KeyError, fmri
# we have no cached copy; save one
# don't specify excludes so on-disk copy has
# all variants
self.set_content(contents)
self.__finiload()
self.__storeback()
if excludes:
self.set_content(contents, excludes)
return
# we have a cached copy of the manifest
mdpath = self.__file_path("manifest.dircache")
# have we computed the dircache?
if not os.path.exists(mdpath): # we're adding cache
self.excludes = EmptyI # to existing manifest
self.__load()
self.__storeback()
if excludes:
self.excludes = excludes
self.__load()
def __load(self):
"""Load all manifest contents from on-disk copy of manifest"""
f = file(self.__file_path("manifest"))
data = f.read()
f.close()
self.set_content(data, self.excludes)
self.__finiload()
def __finiload(self):
"""Finish loading.... this part of initialization is common
to multiple code paths"""
self.loaded = True
# this needs to change; we should not modify on-disk manifest
if "publisher" not in self.attributes:
if not self.fmri.has_publisher():
pub = self.__pub
else:
pub = self.fmri.get_publisher()
Manifest.__setitem__(self, "publisher", pub)
def __storeback(self):
""" store the current action set; also create per-type
caches"""
assert self.loaded
try:
self.store(self.__file_path("manifest"))
self.__storebytype()
except EnvironmentError, e:
# this allows us to try to cache new manifests
# when non-root w/o failures
if e.errno not in (errno.EROFS, errno.EACCES):
raise
def __storebytype(self):
""" create manifest.<typename> files to accelerate partial
parsing of manifests. Separate from __storeback code to
allow upgrade to reuse existing on disk manifests"""
assert self.loaded
# create per-action type cache; use rename to avoid
# corrupt files if ^C'd in the middle
# XXX consider use of per-process tmp file names
for n in self.actions_bytype.keys():
f = file(self.__file_path("manifest.%s.tmp" % n),
"w")
for a in self.actions_bytype[n]:
f.write("%s\n" % a)
f.close()
os.rename(self.__file_path("manifest.%s.tmp" % n),
self.__file_path("manifest.%s" % n))
# create dircache
f = file(self.__file_path("manifest.dircache.tmp"), "w")
dirs = self.__actions_to_dirs()
for s in self.__gen_dirs_to_str(dirs):
f.write(s)
f.close()
os.rename(self.__file_path("manifest.dircache.tmp"),
self.__file_path("manifest.dircache"))
def __gen_dirs_to_str(self, dirs):
""" from a dictionary of paths, generate contents of dircache
file"""
for d in dirs:
for v in dirs[d]:
yield "dir path=%s %s\n" % \
(d, " ".join("%s=%s" % t \
for t in v.iteritems()))
def __actions_to_dirs(self):
""" create dictionary of all directories referenced
by actions explicitly or implicitly from self.actions...
include variants as values; collapse variants where possible"""
assert self.loaded
dirs = {}
# build a dictionary containing all directories tagged w/
# variants
for a in self.actions:
v, f = a.get_varcet_keys()
variants = dict((name, a.attrs[name]) for name in v + f)
for d in expanddirs(a.directory_references()):
if d not in dirs:
dirs[d] = [variants]
elif variants not in dirs[d]:
dirs[d].append(variants)
# remove any tags if any entries are always installed (NULL)
for d in dirs:
if {} in dirs[d]:
dirs[d] = [{}]
continue
# could collapse dirs where all variants are present
return dirs
def get_directories(self, excludes):
""" return a list of directories implicitly or
explicitly referenced by this object"""
mpath = self.__file_path("manifest.dircache")
if not os.path.exists(mpath):
# no cached copy
if not self.loaded:
# need to load from disk
self.__load()
# generate actions that contain directories
alist = [
actions.fromstr(s)
for s in self.__gen_dirs_to_str(
self.__actions_to_dirs(self))
]
else:
# we have cached copy on disk; use it
f = file(mpath)
alist = [actions.fromstr(s.strip()) for s in f]
f.close()
s = set([
a.attrs["path"]
for a in alist
if a.include_this(excludes)
])
return list(s)
def gen_actions_by_type(self, atype, excludes=EmptyI):
""" generate actions of the specified type;
use already in-memory stuff if already loaded,
otherwise use per-action types files"""
if self.loaded: #if already loaded, use in-memory cached version
# invoke subclass method to generate action by action
for a in Manifest.gen_actions_by_type(self, atype,
excludes):
yield a
return
mpath = self.__file_path("manifest.dircache")
if not os.path.exists(mpath):
# no cached copy :-(
if not self.loaded:
# get manifest from disk
self.__load()
# invoke subclass method to generate action by action
for a in Manifest.gen_actions_by_type(self, atype,
excludes):
yield a
else:
# we have a cached copy - use it
mpath = self.__file_path("manifest.%s" % atype)
if not os.path.exists(mpath):
return # no such action in this manifest
f = file(mpath)
for l in f:
a = actions.fromstr(l.strip())
if a.include_this(excludes):
yield a
f.close()
def __load_attributes(self):
"""Load attributes dictionary from cached set actions;
this speeds up pkg info a lot"""
mpath = self.__file_path("manifest.set")
if not os.path.exists(mpath):
return False
f = file(mpath)
for l in f:
a = actions.fromstr(l.strip())
if a.include_this(self.excludes):
self.fill_attributes(a)
f.close()
return True
def __getitem__(self, key):
if not self.loaded and not self.__load_attributes():
self.__load()
return Manifest.__getitem__(self, key)
def __setitem__(self, key, value):
"""No assignments to cached manifests allowed."""
assert "CachedManifests are not dicts"
def __contains__(self, key):
if not self.loaded and not self.__load_attributes():
self.__load()
return Manifest.__contains__(self, key)
def get(self, key, default):
try:
return self[key]
except KeyError:
return default
def get_variants(self, name):
if not self.loaded and not self.__load_attributes():
self.__load()
return Manifest.get_variants(self, name)
@staticmethod
def search_dict(file_path, excludes, return_line=False):
if not self.loaded:
self.__load()
return Manifest.search_dict(file_path, excludes,
return_line=return_line)
def gen_actions(self, excludes=EmptyI):
if not self.loaded:
self.__load()
return Manifest.gen_actions(self, excludes=excludes)
def duplicates(self, excludes=EmptyI):
if not self.loaded:
self.__load()
return Manifest.duplicates(self, excludes=excludes)
def difference(self, origin, origin_exclude=EmptyI, self_exclude=EmptyI):
if not self.loaded:
self.__load()
return Manifest.difference(self, origin,
origin_exclude=origin_exclude,
self_exclude=self_exclude)