1516N/A#!/usr/bin/python
23N/A#
23N/A# CDDL HEADER START
23N/A#
23N/A# The contents of this file are subject to the terms of the
23N/A# Common Development and Distribution License (the "License").
23N/A# You may not use this file except in compliance with the License.
23N/A#
23N/A# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
23N/A# or http://www.opensolaris.org/os/licensing.
23N/A# See the License for the specific language governing permissions
23N/A# and limitations under the License.
23N/A#
23N/A# When distributing Covered Code, include this CDDL HEADER in each
23N/A# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
23N/A# If applicable, add the following below this CDDL HEADER, with the
23N/A# fields enclosed by brackets "[]" replaced with your own identifying
23N/A# information: Portions Copyright [yyyy] [name of copyright owner]
23N/A#
23N/A# CDDL HEADER END
23N/A#
3339N/A# Copyright (c) 2007, 2016, Oracle and/or its affiliates. All rights reserved.
23N/A
221N/A"""Interfaces and implementation for the Catalog object, as well as functions
221N/Athat operate on lists of package FMRIs."""
221N/A
3339N/Afrom __future__ import print_function
1352N/Aimport copy
1352N/Aimport calendar
2510N/Aimport collections
429N/Aimport datetime
1352N/Aimport errno
2510N/Aimport fnmatch
1516N/Aimport hashlib
1352N/Aimport os
1352N/Aimport simplejson as json
3234N/Aimport six
635N/Aimport stat
1352N/Aimport threading
2810N/Aimport types
23N/A
3339N/Afrom collections import OrderedDict
3234N/Afrom operator import itemgetter
3234N/A
1369N/Aimport pkg.actions
1352N/Aimport pkg.client.api_errors as api_errors
2616N/Aimport pkg.client.pkgdefs as pkgdefs
23N/Aimport pkg.fmri as fmri
1463N/Aimport pkg.misc as misc
429N/Aimport pkg.portable as portable
1352N/Aimport pkg.version
1352N/A
2510N/Afrom pkg.misc import EmptyDict, EmptyI
1352N/A
1352N/Aclass _JSONWriter(object):
1352N/A """Private helper class used to serialize catalog data and generate
1352N/A signatures."""
1352N/A
1833N/A def __init__(self, data, single_pass=False, pathname=None, sign=True):
1352N/A self.__data = data
1352N/A self.__fileobj = None
1358N/A
1833N/A # Determines whether data is encoded in a single pass (uses
1833N/A # more memory) or iteratively.
1833N/A self.__single_pass = single_pass
1833N/A
1463N/A # Default to a 32K buffer.
2810N/A self.__bufsz = 32 * 1024
1463N/A
2962N/A # catalog signatures *must* use sha-1 only since clients
2962N/A # compare entire dictionaries against the reported hash from
2962N/A # the catalog in the various <CatalogPartBase>.validate()
2962N/A # methods rather than just attributes within those dictionaries.
2962N/A # If old clients are to interoperate with new repositories, the
2962N/A # computed and expected dictionaries must be identical at
2962N/A # present, so we must use sha-1.
1358N/A if sign:
1463N/A if not pathname:
1463N/A # Only needed if not writing to __fileobj.
1516N/A self.__sha_1 = hashlib.sha1()
1463N/A self.__sha_1_value = None
1463N/A
1358N/A self.__sign = sign
1431N/A self.pathname = pathname
1431N/A
1431N/A if not pathname:
1431N/A return
1431N/A
1463N/A # Call statvfs to find optimal blocksize for destination.
1463N/A dest_dir = os.path.dirname(self.pathname)
1431N/A try:
1463N/A destvfs = os.statvfs(dest_dir)
1463N/A # Set the file buffer size to the blocksize of our
1463N/A # filesystem.
3245N/A self.__bufsz = destvfs.f_bsize
3171N/A except EnvironmentError as e:
1463N/A if e.errno == errno.EACCES:
1463N/A raise api_errors.PermissionsException(
1463N/A e.filename)
3171N/A except AttributeError as e:
1463N/A # os.statvfs is not available on some platforms.
1463N/A pass
1463N/A
1463N/A try:
1463N/A tfile = open(pathname, "wb", self.__bufsz)
3171N/A except EnvironmentError as e:
1431N/A if e.errno == errno.EACCES:
1431N/A raise api_errors.PermissionsException(
1431N/A e.filename)
1431N/A if e.errno == errno.EROFS:
1431N/A raise api_errors.ReadOnlyFileSystemException(
1431N/A e.filename)
1431N/A raise
1431N/A self.__fileobj = tfile
1352N/A
1352N/A def signatures(self):
1352N/A """Returns a dictionary mapping digest algorithms to the
1352N/A hex-encoded digest values of the text of the catalog."""
1352N/A
1358N/A if not self.__sign:
1358N/A return {}
1463N/A return { "sha-1": self.__sha_1_value }
1352N/A
1833N/A def _dump(self, obj, fp, skipkeys=False, ensure_ascii=True,
1833N/A check_circular=True, allow_nan=True, cls=json.JSONEncoder,
1833N/A indent=None, separators=None, encoding='utf-8', default=None, **kw):
1833N/A iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
1833N/A check_circular=check_circular, allow_nan=allow_nan,
1833N/A indent=indent, separators=separators, encoding=encoding,
1833N/A default=default, **kw).iterencode(obj,
1833N/A _one_shot=self.__single_pass)
3339N/A fp.writelines(misc.force_bytes(i) for i in iterable)
1833N/A
1352N/A def save(self):
1352N/A """Serializes and stores the provided data in JSON format."""
1358N/A
1352N/A # sort_keys is necessary to ensure consistent signature
1352N/A # generation. It has a minimal performance cost as well (on
1358N/A # on SPARC and x86), so shouldn't be an issue. However, it
1358N/A # is only needed if the caller has indicated that the content
1358N/A # should be signed.
1463N/A
1463N/A # Whenever possible, avoid using the write wrapper (self) as
1463N/A # this can greatly increase write times.
1463N/A out = self.__fileobj
1463N/A if not out:
1463N/A out = self
1463N/A
1833N/A self._dump(self.__data, out, check_circular=False,
1463N/A separators=(",", ":"), sort_keys=self.__sign)
3339N/A out.write(b"\n")
1352N/A
1352N/A if self.__fileobj:
1352N/A self.__fileobj.close()
1463N/A
1463N/A if not self.__sign or not self.__fileobj:
1463N/A # Can't sign unless a file object is provided. And if
1463N/A # one is provided, but no signing is to be done, then
1463N/A # ensure the fileobject is discarded.
1352N/A self.__fileobj = None
1463N/A if self.__sign:
1463N/A self.__sha_1_value = self.__sha_1.hexdigest()
1463N/A return
1463N/A
1463N/A # Ensure file object goes out of scope.
1463N/A self.__fileobj = None
1463N/A
1463N/A # Calculating sha-1 this way is much faster than intercepting
1463N/A # write calls because of the excessive number of write calls
1463N/A # that json.dump() triggers (1M+ for /dev catalog files).
2962N/A self.__sha_1_value = misc.get_data_digest(self.pathname,
2962N/A hash_func=hashlib.sha1)[0]
1463N/A
1463N/A # Open the JSON file so that the signature data can be added.
3339N/A with open(self.pathname, "rb+", self.__bufsz) as sfile:
3339N/A # The last bytes should be "}\n", which is where the
3339N/A # signature data structure needs to be appended.
3339N/A sfile.seek(-2, os.SEEK_END)
3339N/A
3339N/A # Add the signature data and close.
3339N/A sfoffset = sfile.tell()
3339N/A if sfoffset > 1:
3339N/A # Catalog is not empty, so a separator is needed.
3339N/A sfile.write(b",")
3339N/A sfile.write(b'"_SIGNATURE":')
3339N/A self._dump(self.signatures(), sfile, check_circular=False,
3339N/A separators=(",", ":"))
3339N/A sfile.write(b"}\n")
1352N/A
1352N/A def write(self, data):
1352N/A """Wrapper function that should not be called by external
1352N/A consumers."""
1358N/A
1358N/A if self.__sign:
1358N/A self.__sha_1.update(data)
1352N/A
1833N/A def writelines(self, iterable):
1833N/A """Wrapper function that should not be called by external
1833N/A consumers."""
1833N/A
1833N/A for l in iterable:
1833N/A self.__sha_1.update(l)
1833N/A
1369N/A
1352N/Aclass CatalogPartBase(object):
1352N/A """A CatalogPartBase object is an abstract class containing core
1352N/A functionality shared between CatalogPart and CatalogAttrs."""
1352N/A
1352N/A # The file mode to be used for all catalog files.
1352N/A __file_mode = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH
1352N/A
1352N/A __meta_root = None
3362N/A __file_root = None
1352N/A last_modified = None
1352N/A loaded = False
1352N/A name = None
1358N/A sign = True
1352N/A signatures = None
1352N/A
3362N/A def __init__(self, name, meta_root=None, sign=True, file_root=None):
1352N/A """Initializes a CatalogPartBase object."""
1352N/A
1352N/A self.meta_root = meta_root
3362N/A self.file_root = file_root
3362N/A if not self.file_root:
3362N/A self.file_root = meta_root
2810N/A # Sanity check: part names can't be pathname-ish.
2810N/A if name != os.path.basename(name):
3362N/A raise api_errors.UnrecognizedCatalogPart(name)
1352N/A self.name = name
1358N/A self.sign = sign
1352N/A self.signatures = {}
157N/A
1352N/A if not self.meta_root or not self.exists:
1352N/A # Operations shouldn't attempt to load the part data
1352N/A # unless meta_root is defined and the data exists.
1352N/A self.loaded = True
1352N/A self.last_modified = datetime.datetime.utcnow()
1352N/A else:
1352N/A self.last_modified = self.__last_modified()
1352N/A
1352N/A @staticmethod
1352N/A def _gen_signatures(data):
1352N/A f = _JSONWriter(data)
1352N/A f.save()
1352N/A return f.signatures()
1352N/A
1352N/A def __get_meta_root(self):
1352N/A return self.__meta_root
1352N/A
3362N/A def __get_file_root(self):
3362N/A return self.__file_root
3362N/A
1352N/A def __last_modified(self):
1352N/A """A UTC datetime object representing the time the file used to
1352N/A to store object metadata was modified, or None if it does not
1352N/A exist yet."""
1352N/A
1352N/A if not self.exists:
1352N/A return None
1352N/A
1352N/A try:
1352N/A mod_time = os.stat(self.pathname).st_mtime
3171N/A except EnvironmentError as e:
1352N/A if e.errno == errno.ENOENT:
1352N/A return None
1352N/A raise
1352N/A return datetime.datetime.utcfromtimestamp(mod_time)
1352N/A
1352N/A def __set_meta_root(self, path):
1352N/A if path:
1352N/A path = os.path.abspath(path)
1352N/A self.__meta_root = path
1352N/A
3362N/A def __set_file_root(self, path):
3362N/A if path:
3362N/A path = os.path.abspath(path)
3362N/A self.__file_root = path
3362N/A
1352N/A def destroy(self):
1431N/A """Removes any on-disk files that exist for the catalog part and
1352N/A discards all content."""
1352N/A
1352N/A if self.pathname:
1352N/A if os.path.exists(self.pathname):
1352N/A try:
1352N/A portable.remove(self.pathname)
3171N/A except EnvironmentError as e:
1431N/A if e.errno == errno.EACCES:
1431N/A raise api_errors.PermissionsException(
1431N/A e.filename)
1431N/A if e.errno == errno.EROFS:
1431N/A raise api_errors.ReadOnlyFileSystemException(
1431N/A e.filename)
1431N/A raise
1352N/A self.signatures = {}
1352N/A self.loaded = False
1352N/A self.last_modified = None
1352N/A
1352N/A @property
1352N/A def exists(self):
1352N/A """A boolean value indicating wheher a file for the catalog part
1352N/A exists at <self.meta_root>/<self.name>."""
1352N/A
1352N/A if not self.pathname:
1352N/A return False
1352N/A return os.path.exists(self.pathname)
1352N/A
1352N/A def load(self):
1352N/A """Load the serialized data for the catalog part and return the
1352N/A resulting structure."""
1352N/A
1352N/A location = os.path.join(self.meta_root, self.name)
1352N/A
1352N/A try:
3362N/A fobj = misc.open_image_file(self.file_root, location,
3362N/A os.O_RDONLY, misc.PKG_FILE_MODE)
3171N/A except EnvironmentError as e:
1352N/A if e.errno == errno.ENOENT:
1352N/A raise api_errors.RetrievalError(e,
1352N/A location=location)
1431N/A if e.errno == errno.EROFS:
1431N/A raise api_errors.ReadOnlyFileSystemException(
1431N/A e.filename)
1431N/A if e.errno == errno.EACCES:
1352N/A raise api_errors.PermissionsException(
1352N/A e.filename)
3362N/A if e.errno == errno.EREMOTE:
3362N/A raise api_errors.UnrecognizedCatalogPart(
3362N/A self.name)
1431N/A raise
1352N/A
1352N/A try:
1352N/A struct = json.load(fobj)
3171N/A except EnvironmentError as e:
1352N/A raise api_errors.RetrievalError(e)
3171N/A except ValueError as e:
1352N/A # Not a valid catalog file.
1658N/A raise api_errors.InvalidCatalogFile(location)
1352N/A
1352N/A self.loaded = True
1358N/A # Signature data, if present, should be removed from the struct
1358N/A # on load and then stored in the signatures object property.
1358N/A self.signatures = struct.pop("_SIGNATURE", {})
3339N/A fobj.close()
1352N/A return struct
1352N/A
1352N/A @property
1352N/A def pathname(self):
1352N/A """The absolute path of the file used to store the data for
1352N/A this part or None if meta_root or name is not set."""
1352N/A
1352N/A if not self.meta_root or not self.name:
1352N/A return None
1352N/A return os.path.join(self.meta_root, self.name)
1352N/A
1833N/A def save(self, data, single_pass=False):
1352N/A """Serialize and store the transformed catalog part's 'data' in
1352N/A a file using the pathname <self.meta_root>/<self.name>.
1352N/A
1833N/A 'data' must be a dict.
1833N/A
1833N/A 'single_pass' is an optional boolean indicating whether the data
1833N/A should be serialized in a single pass. This is significantly
1833N/A faster, but requires that the entire set of data be serialized
1833N/A in-memory instead of iteratively writing it to the target
1833N/A storage object."""
1833N/A
1833N/A f = _JSONWriter(data, single_pass=single_pass,
1833N/A pathname=self.pathname, sign=self.sign)
1352N/A f.save()
1352N/A
1352N/A # Update in-memory copy to reflect stored data.
1352N/A self.signatures = f.signatures()
1352N/A
1431N/A # Ensure the permissions on the new file are correct.
1352N/A try:
1431N/A os.chmod(self.pathname, self.__file_mode)
3171N/A except EnvironmentError as e:
1431N/A if e.errno == errno.EACCES:
1431N/A raise api_errors.PermissionsException(
1431N/A e.filename)
1431N/A if e.errno == errno.EROFS:
1431N/A raise api_errors.ReadOnlyFileSystemException(
1431N/A e.filename)
1352N/A raise
1352N/A
1352N/A # Finally, set the file times to match the last catalog change.
1352N/A if self.last_modified:
1352N/A mtime = calendar.timegm(
1352N/A self.last_modified.utctimetuple())
1431N/A os.utime(self.pathname, (mtime, mtime))
1352N/A
1352N/A meta_root = property(__get_meta_root, __set_meta_root)
3362N/A file_root = property(__get_file_root, __set_file_root)
23N/A
258N/A
1352N/Aclass CatalogPart(CatalogPartBase):
1352N/A """A CatalogPart object is the representation of a subset of the package
1352N/A FMRIs available from a package repository."""
1352N/A
1352N/A __data = None
1352N/A ordered = None
1352N/A
3362N/A def __init__(self, name, meta_root=None, ordered=True, sign=True,
3362N/A file_root=None):
1352N/A """Initializes a CatalogPart object."""
1352N/A
1352N/A self.__data = {}
1352N/A self.ordered = ordered
2810N/A if not name.startswith("catalog."):
3362N/A raise api_errors.UnrecognizedCatalogPart(name)
1358N/A CatalogPartBase.__init__(self, name, meta_root=meta_root,
3362N/A sign=sign, file_root=file_root)
1352N/A
1537N/A def __iter_entries(self, last=False, ordered=False, pubs=EmptyI):
1537N/A """Private generator function to iterate over catalog entries.
1537N/A
1537N/A 'last' is a boolean value that indicates only the last entry
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the CatalogPart has been saved since the last
1537N/A modifying operation, or sort() has has been called, this will
1537N/A also be the newest version of the package.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1352N/A
1352N/A self.load()
1537N/A if ordered:
1537N/A stems = self.pkg_names(pubs=pubs)
1537N/A else:
1537N/A stems = (
1537N/A (pub, stem)
1537N/A for pub in self.publishers(pubs=pubs)
1537N/A for stem in self.__data[pub]
1537N/A )
1537N/A
1537N/A if last:
1537N/A return (
1537N/A (pub, stem, self.__data[pub][stem][-1])
1537N/A for pub, stem in stems
1537N/A )
1537N/A
1537N/A if ordered:
1537N/A return (
1537N/A (pub, stem, entry)
1537N/A for pub, stem in stems
1537N/A for entry in reversed(self.__data[pub][stem])
1537N/A )
1537N/A return (
1537N/A (pub, stem, entry)
1537N/A for pub, stem in stems
1537N/A for entry in self.__data[pub][stem]
1537N/A )
1352N/A
1549N/A def add(self, pfmri=None, metadata=None, op_time=None, pub=None,
1549N/A stem=None, ver=None):
1549N/A """Add a catalog entry for a given FMRI or FMRI components.
1352N/A
1352N/A 'metadata' is an optional dict containing the catalog
1352N/A metadata that should be stored for the specified FMRI.
1352N/A
1352N/A The dict representing the entry is returned to callers,
1352N/A but should not be modified.
1352N/A """
1352N/A
1549N/A assert pfmri or (pub and stem and ver)
1549N/A if pfmri and not pfmri.publisher:
1549N/A raise api_errors.AnarchicalCatalogFMRI(str(pfmri))
1549N/A
1549N/A if not self.loaded:
1549N/A # Hot path, so avoid calling load unless necessary, even
1549N/A # though it performs this check already.
1549N/A self.load()
1549N/A
1549N/A if pfmri:
1549N/A pub, stem, ver = pfmri.tuple()
1549N/A ver = str(ver)
1549N/A
1549N/A pkg_list = self.__data.setdefault(pub, {})
1549N/A ver_list = pkg_list.setdefault(stem, [])
1352N/A for entry in ver_list:
1352N/A if entry["version"] == ver:
1549N/A if not pfmri:
3158N/A pfmri = "pkg://{0}/{1}@{2}".format(pub,
3158N/A stem, ver)
1352N/A raise api_errors.DuplicateCatalogEntry(
1352N/A pfmri, operation="add",
1352N/A catalog_name=self.pathname)
1352N/A
1549N/A if metadata is not None:
1352N/A entry = metadata
1352N/A else:
1352N/A entry = {}
1352N/A entry["version"] = ver
1352N/A
1352N/A ver_list.append(entry)
1352N/A if self.ordered:
1463N/A self.sort(pfmris=set([pfmri]))
1352N/A
1352N/A if not op_time:
1352N/A op_time = datetime.datetime.utcnow()
1352N/A self.last_modified = op_time
1352N/A self.signatures = {}
1352N/A return entry
1352N/A
1606N/A def destroy(self):
1606N/A """Removes any on-disk files that exist for the catalog part and
1606N/A discards all content."""
1606N/A
1606N/A self.__data = {}
1606N/A return CatalogPartBase.destroy(self)
1606N/A
1537N/A def entries(self, cb=None, last=False, ordered=False, pubs=EmptyI):
1352N/A """A generator function that produces tuples of the form
1352N/A (fmri, entry) as it iterates over the contents of the catalog
1352N/A part (where entry is the related catalog entry for the fmri).
1352N/A Callers should not modify any of the data that is returned.
1352N/A
1537N/A 'cb' is an optional callback function that will be executed for
1537N/A each package. It must accept two arguments: 'pkg' and 'entry'.
1537N/A 'pkg' is an FMRI object and 'entry' is the dictionary structure
1537N/A of the catalog entry for the package. If the callback returns
1537N/A False, then the entry will not be included in the results.
1537N/A
1537N/A 'last' is a boolean value that indicates only the last entry
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the CatalogPart has been saved since the last
1537N/A modifying operation, or sort() has has been called, this will
1537N/A also be the newest version of the package.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to.
1537N/A
1352N/A Results are always in catalog version order on a per-
1352N/A publisher, per-stem basis.
1352N/A """
1352N/A
1537N/A for pub, stem, entry in self.__iter_entries(last=last,
1537N/A ordered=ordered, pubs=pubs):
2639N/A f = fmri.PkgFmri(name=stem, publisher=pub,
2639N/A version=entry["version"])
1537N/A if cb is None or cb(f, entry):
1537N/A yield f, entry
1537N/A
1537N/A def entries_by_version(self, name, pubs=EmptyI):
1352N/A """A generator function that produces tuples of (version,
1352N/A entries), where entries is a list of tuples of the format
1352N/A (fmri, entry) where entry is the catalog entry for the
1537N/A FMRI) as it iterates over the CatalogPart contents.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1352N/A
1352N/A self.load()
1352N/A
1352N/A versions = {}
1352N/A entries = {}
1537N/A for pub in self.publishers(pubs=pubs):
1352N/A ver_list = self.__data[pub].get(name, ())
1352N/A for entry in ver_list:
1352N/A sver = entry["version"]
2639N/A pfmri = fmri.PkgFmri(name=name, publisher=pub,
2639N/A version=sver)
1352N/A
1352N/A versions[sver] = pfmri.version
1352N/A entries.setdefault(sver, [])
1352N/A entries[sver].append((pfmri, entry))
1352N/A
3234N/A for key, ver in sorted(six.iteritems(versions), key=itemgetter(1)):
1352N/A yield ver, entries[key]
1352N/A
1537N/A def fmris(self, last=False, objects=True, ordered=False, pubs=EmptyI):
1537N/A """A generator function that produces FMRIs as it iterates
1537N/A over the contents of the catalog part.
1537N/A
1537N/A 'last' is a boolean value that indicates only the last fmri
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the CatalogPart has been saved since the last
1537N/A modifying operation, or sort() has has been called, this will
1537N/A also be the newest version of the package.
1352N/A
1352N/A 'objects' is an optional boolean value indicating whether
1352N/A FMRIs should be returned as FMRI objects or as strings.
1352N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to.
1537N/A
1352N/A Results are always in catalog version order on a per-
1537N/A publisher, per-stem basis."""
1352N/A
1352N/A if objects:
1537N/A for pub, stem, entry in self.__iter_entries(last=last,
1537N/A ordered=ordered, pubs=pubs):
2639N/A yield fmri.PkgFmri(name=stem, publisher=pub,
2639N/A version=entry["version"])
1352N/A return
1352N/A
1537N/A for pub, stem, entry in self.__iter_entries(last=last,
1537N/A ordered=ordered, pubs=pubs):
3158N/A yield "pkg://{0}/{1}@{2}".format(pub,
1352N/A stem, entry["version"])
1352N/A return
1352N/A
1537N/A def fmris_by_version(self, name, pubs=EmptyI):
1352N/A """A generator function that produces tuples of (version,
1352N/A fmris), where fmris is a list of the fmris related to the
1537N/A version.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1352N/A
1352N/A self.load()
1352N/A
1352N/A versions = {}
1352N/A entries = {}
1537N/A for pub in self.publishers(pubs=pubs):
1537N/A ver_list = self.__data[pub].get(name, None)
1549N/A if not ver_list:
1352N/A continue
1352N/A
1352N/A for entry in ver_list:
1352N/A sver = entry["version"]
2639N/A pfmri = fmri.PkgFmri(name=name, publisher=pub,
2639N/A version=sver)
1352N/A
1352N/A versions[sver] = pfmri.version
1352N/A entries.setdefault(sver, [])
1352N/A entries[sver].append(pfmri)
1352N/A
3234N/A for key, ver in sorted(six.iteritems(versions), key=itemgetter(1)):
1352N/A yield ver, entries[key]
1352N/A
1537N/A def get_entry(self, pfmri=None, pub=None, stem=None, ver=None):
1549N/A """Returns the catalog part entry for the given package FMRI or
1537N/A FMRI components."""
1537N/A
1549N/A assert pfmri or (pub and stem and ver)
1549N/A if pfmri and not pfmri.publisher:
1549N/A raise api_errors.AnarchicalCatalogFMRI(str(pfmri))
1352N/A
1537N/A # Since this is a hot path, this function checks for loaded
1537N/A # status before attempting to call the load function.
1537N/A if not self.loaded:
1537N/A self.load()
1537N/A
1537N/A if pfmri:
1537N/A pub, stem, ver = pfmri.tuple()
1537N/A ver = str(ver)
1537N/A
1537N/A pkg_list = self.__data.get(pub, None)
1549N/A if not pkg_list:
1537N/A return
1537N/A
1537N/A ver_list = pkg_list.get(stem, ())
1369N/A for entry in ver_list:
1369N/A if entry["version"] == ver:
1369N/A return entry
1352N/A
1352N/A def get_package_counts(self):
1352N/A """Returns a tuple of integer values (package_count,
1352N/A package_version_count). The first is the number of
1352N/A unique packages (per-publisher), and the second is the
1352N/A number of unique package versions (per-publisher and
1352N/A stem)."""
1352N/A
1352N/A self.load()
1352N/A package_count = 0
1352N/A package_version_count = 0
1431N/A for pub in self.publishers():
1352N/A for stem in self.__data[pub]:
1352N/A package_count += 1
1352N/A package_version_count += \
1352N/A len(self.__data[pub][stem])
1352N/A return (package_count, package_version_count)
1352N/A
3110N/A def get_package_counts_by_pub(self, pubs=EmptyI):
1968N/A """Returns a generator of tuples of the form (pub,
1968N/A package_count, package_version_count). 'pub' is the publisher
1968N/A prefix, 'package_count' is the number of unique packages for the
1968N/A publisher, and 'package_version_count' is the number of unique
1968N/A package versions for the publisher.
1968N/A """
1968N/A
1968N/A self.load()
3110N/A for pub in self.publishers(pubs=pubs):
1968N/A package_count = 0
1968N/A package_version_count = 0
1968N/A for stem in self.__data[pub]:
1968N/A package_count += 1
1968N/A package_version_count += \
1968N/A len(self.__data[pub][stem])
1968N/A yield pub, package_count, package_version_count
1968N/A
1352N/A def load(self):
1352N/A """Load and transform the catalog part's data, preparing it
1352N/A for use."""
1352N/A
1352N/A if self.loaded:
1352N/A # Already loaded, or only in-memory.
1352N/A return
1352N/A self.__data = CatalogPartBase.load(self)
1352N/A
1537N/A def names(self, pubs=EmptyI):
1352N/A """Returns a set containing the names of all the packages in
1537N/A the CatalogPart.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1352N/A
1352N/A self.load()
1352N/A return set((
1352N/A stem
1537N/A for pub in self.publishers(pubs=pubs)
1352N/A for stem in self.__data[pub]
1352N/A ))
1352N/A
1537N/A def pkg_names(self, pubs=EmptyI):
1537N/A """A generator function that produces package tuples of the form
1537N/A (pub, stem) as it iterates over the contents of the CatalogPart.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
2547N/A the results to. If specified, publishers will be sorted in
2547N/A the order given.
1537N/A
1537N/A Results are always returned sorted by stem and then by
1537N/A publisher."""
1537N/A
1537N/A self.load()
1537N/A
1537N/A # Results have to be sorted by stem first, and by
1537N/A # publisher prefix second.
1537N/A pkg_list = [
3158N/A "{0}!{1}".format(stem, pub)
1537N/A for pub in self.publishers(pubs=pubs)
1537N/A for stem in self.__data[pub]
1537N/A ]
1537N/A
2547N/A pub_sort = None
2547N/A if pubs:
2547N/A pos = dict((p, i) for (i, p) in enumerate(pubs))
3245N/A def pub_key(a):
2547N/A astem, apub = a.split("!", 1)
3245N/A return (astem, pos[apub])
3245N/A pub_sort = pub_key
3245N/A
3245N/A for entry in sorted(pkg_list, key=pub_sort):
1537N/A stem, pub = entry.split("!", 1)
1537N/A yield pub, stem
1537N/A
1463N/A def publishers(self, pubs=EmptyI):
1431N/A """A generator function that returns publisher prefixes as it
1463N/A iterates over the package data in the CatalogPart.
1463N/A
1463N/A 'pubs' is an optional list that contains the prefixes of the
1463N/A publishers to restrict the results to."""
1431N/A
1431N/A self.load()
1431N/A for pub in self.__data:
1463N/A # Any entries starting with "_" are part of the
1431N/A # reserved catalog namespace.
1463N/A if not pub[0] == "_" and (not pubs or pub in pubs):
1431N/A yield pub
1431N/A
1352N/A def remove(self, pfmri, op_time=None):
1352N/A """Remove a package and its metadata."""
1352N/A
1352N/A if not pfmri.publisher:
1352N/A raise api_errors.AnarchicalCatalogFMRI(pfmri.get_fmri())
1352N/A
1352N/A self.load()
1537N/A pkg_list = self.__data.get(pfmri.publisher, None)
1549N/A if not pkg_list:
1352N/A raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1352N/A
1352N/A ver = str(pfmri.version)
1352N/A ver_list = pkg_list.get(pfmri.pkg_name, [])
1352N/A for i, entry in enumerate(ver_list):
1352N/A if entry["version"] == ver:
1352N/A # Safe to do this since a 'break' is done
1352N/A # immediately after removals are performed.
1352N/A del ver_list[i]
1352N/A if not ver_list:
1352N/A # When all version entries for a
1352N/A # package are removed, its stem
1352N/A # should be also.
1352N/A del pkg_list[pfmri.pkg_name]
1352N/A if not pkg_list:
1352N/A # When all package stems for a
1352N/A # publisher have been removed,
1352N/A # it should be also.
1352N/A del self.__data[pfmri.publisher]
1352N/A break
1352N/A else:
1352N/A raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1352N/A
1352N/A if not op_time:
1352N/A op_time = datetime.datetime.utcnow()
1352N/A self.last_modified = op_time
1352N/A self.signatures = {}
1352N/A
1833N/A def save(self, single_pass=False):
1352N/A """Transform and store the catalog part's data in a file using
1833N/A the pathname <self.meta_root>/<self.name>.
1833N/A
1833N/A 'single_pass' is an optional boolean indicating whether the data
1833N/A should be serialized in a single pass. This is significantly
1833N/A faster, but requires that the entire set of data be serialized
1833N/A in-memory instead of iteratively writing it to the target
1833N/A storage object."""
1352N/A
1352N/A if not self.meta_root:
1352N/A # Assume this is in-memory only.
1352N/A return
1833N/A
2022N/A # Ensure content is loaded before attempting save.
2022N/A self.load()
2022N/A
1833N/A CatalogPartBase.save(self, self.__data, single_pass=single_pass)
1352N/A
1463N/A def sort(self, pfmris=None, pubs=None):
1352N/A """Re-sorts the contents of the CatalogPart such that version
1352N/A entries for each package stem are in ascending order.
1352N/A
1463N/A 'pfmris' is an optional set of FMRIs to restrict the sort to.
1463N/A This is useful during catalog operations as only entries for
1463N/A the corresponding package stem(s) need to be sorted.
1463N/A
1463N/A 'pubs' is an optional set of publisher prefixes to restrict
1463N/A the sort to. This is useful during catalog operations as only
1463N/A entries for the corresponding publisher stem(s) need to be
1463N/A sorted. This option has no effect if 'pfmris' is also
1463N/A provided.
1463N/A
1463N/A If neither 'pfmris' or 'pubs' is provided, all entries will be
1463N/A sorted."""
1352N/A
3245N/A def key_func(item):
3245N/A return pkg.version.Version(item["version"])
1352N/A
1352N/A self.load()
1463N/A if pfmris is not None:
1463N/A processed = set()
1463N/A for f in pfmris:
1463N/A pkg_stem = f.get_pkg_stem()
1463N/A if pkg_stem in processed:
1463N/A continue
1463N/A processed.add(pkg_stem)
1463N/A
1463N/A # The specified FMRI may not exist in this
1463N/A # CatalogPart, so continue if it does not
1463N/A # exist.
1463N/A pkg_list = self.__data.get(f.publisher, None)
1549N/A if pkg_list:
1463N/A ver_list = pkg_list.get(f.pkg_name,
1463N/A None)
1549N/A if ver_list:
3245N/A ver_list.sort(key=key_func)
1352N/A return
1352N/A
1463N/A for pub in self.publishers(pubs=pubs):
1352N/A for stem in self.__data[pub]:
3245N/A self.__data[pub][stem].sort(key=key_func)
1352N/A
1537N/A def tuples(self, last=False, ordered=False, pubs=EmptyI):
1537N/A """A generator function that produces FMRI tuples as it
1537N/A iterates over the contents of the catalog part.
1537N/A
1537N/A 'last' is a boolean value that indicates only the last FMRI
1537N/A tuple for each package on a per-publisher basis should be
1537N/A returned. As long as the CatalogPart has been saved since
1537N/A the last modifying operation, or sort() has has been called,
1537N/A this will also be the newest version of the package.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1537N/A
1537N/A return (
1537N/A (pub, stem, entry["version"])
1537N/A for pub, stem, entry in self.__iter_entries(last=last,
1537N/A ordered=ordered, pubs=pubs)
1537N/A )
1537N/A
1537N/A def tuple_entries(self, cb=None, last=False, ordered=False, pubs=EmptyI):
1537N/A """A generator function that produces tuples of the form ((pub,
1537N/A stem, version), entry) as it iterates over the contents of the
1537N/A catalog part (where entry is the related catalog entry for the
1537N/A fmri). Callers should not modify any of the data that is
1537N/A returned.
1537N/A
1537N/A 'cb' is an optional callback function that will be executed for
1537N/A each package. It must accept two arguments: 'pkg' and 'entry'.
1537N/A 'pkg' is an FMRI tuple and 'entry' is the dictionary structure
1537N/A of the catalog entry for the package. If the callback returns
1537N/A False, then the entry will not be included in the results.
1537N/A
1537N/A 'last' is a boolean value that indicates only the last entry
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the CatalogPart has been saved since the last
1537N/A modifying operation, or sort() has has been called, this will
1537N/A also be the newest version of the package.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to.
1537N/A
1537N/A Results are always in catalog version order on a per-publisher,
1537N/A per-stem basis."""
1537N/A
1537N/A for pub, stem, entry in self.__iter_entries(last=last,
1537N/A ordered=ordered, pubs=pubs):
1537N/A t = (pub, stem, entry["version"])
1537N/A if cb is None or cb(t, entry):
1537N/A yield t, entry
1431N/A
2810N/A def validate(self, signatures=None, require_signatures=False):
1352N/A """Verifies whether the signatures for the contents of the
1352N/A CatalogPart match the specified signature data, or if not
1352N/A provided, the current signature data. Raises the exception
1352N/A named 'BadCatalogSignatures' on failure."""
1352N/A
2810N/A if not self.signatures and not signatures and \
2810N/A not require_signatures:
2810N/A # Nothing to validate, and we're not required to.
1352N/A return
2022N/A
2022N/A # Ensure content is loaded before attempting to retrieve
2022N/A # or generate signature data.
2022N/A self.load()
1352N/A if not signatures:
1352N/A signatures = self.signatures
1352N/A
1352N/A new_signatures = self._gen_signatures(self.__data)
1352N/A if new_signatures != signatures:
1352N/A raise api_errors.BadCatalogSignatures(self.pathname)
1352N/A
814N/A
1352N/Aclass CatalogUpdate(CatalogPartBase):
1352N/A """A CatalogUpdate object is an augmented representation of a subset
1352N/A of the package data contained within a Catalog."""
1352N/A
1352N/A # Properties.
1352N/A __data = None
1352N/A last_modified = None
1352N/A
1352N/A # Operation constants.
1352N/A ADD = "add"
1352N/A REMOVE = "remove"
1352N/A
3362N/A def __init__(self, name, meta_root=None, sign=True, file_root=None):
1352N/A """Initializes a CatalogUpdate object."""
1352N/A
1352N/A self.__data = {}
2810N/A if not name.startswith("update."):
3362N/A raise api_errors.UnrecognizedCatalogPart(name)
1431N/A CatalogPartBase.__init__(self, name, meta_root=meta_root,
3362N/A sign=sign, file_root=file_root)
1352N/A
1352N/A def add(self, pfmri, operation, op_time, metadata=None):
1352N/A """Records the specified catalog operation and any related
1352N/A catalog metadata for the specified package FMRI.
1352N/A
1352N/A 'operation' must be one of the following constant values
1352N/A provided by the CatalogUpdate class:
1352N/A ADD
1352N/A REMOVE
1352N/A
1352N/A 'op_time' is a UTC datetime object indicating the time
1352N/A the catalog operation was performed.
1352N/A
1352N/A 'metadata' is an optional dict containing the catalog
1352N/A metadata that should be stored for the specified FMRI
1352N/A indexed by catalog part (e.g. "dependency", "summary",
1352N/A etc.)."""
1352N/A
1352N/A if not pfmri.publisher:
1352N/A raise api_errors.AnarchicalCatalogFMRI(pfmri.get_fmri())
1352N/A
1352N/A if operation not in (self.ADD, self.REMOVE):
1352N/A raise api_errors.UnknownUpdateType(operation)
1352N/A
1352N/A self.load()
1537N/A self.__data.setdefault(pfmri.publisher, {})
1537N/A pkg_list = self.__data[pfmri.publisher]
1537N/A
1537N/A pkg_list.setdefault(pfmri.pkg_name, [])
1537N/A ver_list = pkg_list[pfmri.pkg_name]
1537N/A
1537N/A if metadata is not None:
1352N/A entry = metadata
1352N/A else:
1352N/A entry = {}
1352N/A entry["op-time"] = datetime_to_basic_ts(op_time)
1352N/A entry["op-type"] = operation
1352N/A entry["version"] = str(pfmri.version)
1352N/A ver_list.append(entry)
1352N/A
1352N/A # To ensure the update log is viewed as having been updated
1352N/A # at the exact same time as the catalog, the last_modified
1352N/A # time of the update log must match the operation time.
1352N/A self.last_modified = op_time
1352N/A self.signatures = {}
1352N/A
1352N/A def load(self):
1352N/A """Load and transform the catalog update's data, preparing it
1352N/A for use."""
1352N/A
1352N/A if self.loaded:
1352N/A # Already loaded, or only in-memory.
1352N/A return
1352N/A self.__data = CatalogPartBase.load(self)
1352N/A
1431N/A def publishers(self):
1431N/A """A generator function that returns publisher prefixes as it
1431N/A iterates over the package data in the CatalogUpdate."""
1431N/A
1431N/A self.load()
1431N/A for pub in self.__data:
1463N/A # Any entries starting with "_" are part of the
1431N/A # reserved catalog namespace.
1463N/A if not pub[0] == "_":
1431N/A yield pub
1431N/A
1352N/A def save(self):
1352N/A """Transform and store the catalog update's data in a file using
1352N/A the pathname <self.meta_root>/<self.name>."""
1352N/A
1352N/A if not self.meta_root:
1352N/A # Assume this is in-memory only.
1352N/A return
2022N/A
2022N/A # Ensure content is loaded before attempting save.
2022N/A self.load()
2022N/A
1352N/A CatalogPartBase.save(self, self.__data)
1352N/A
1352N/A def updates(self):
1352N/A """A generator function that produces tuples of the format
1352N/A (fmri, op_type, op_time, metadata). Where:
1352N/A
1352N/A * 'fmri' is a PkgFmri object for the package.
1352N/A
1352N/A * 'op_type' is a CatalogUpdate constant indicating
1352N/A the catalog operation performed.
1352N/A
1352N/A * 'op_time' is a UTC datetime object representing the
1352N/A time time the catalog operation was performed.
1352N/A
1352N/A * 'metadata' is a dict containing the catalog metadata
1352N/A for the FMRI indexed by catalog part name.
1352N/A
1352N/A Results are always in ascending operation time order on a
1352N/A per-publisher, per-stem basis.
1352N/A """
1352N/A
1352N/A self.load()
1352N/A
1352N/A def get_update(pub, stem, entry):
1352N/A mdata = {}
1352N/A for key in entry:
1352N/A if key.startswith("catalog."):
1352N/A mdata[key] = entry[key]
1352N/A op_time = basic_ts_to_datetime(entry["op-time"])
2639N/A pfmri = fmri.PkgFmri(name=stem, publisher=pub,
2639N/A version=entry["version"])
1352N/A return (pfmri, entry["op-type"], op_time, mdata)
1352N/A
1431N/A for pub in self.publishers():
1352N/A for stem in self.__data[pub]:
1352N/A for entry in self.__data[pub][stem]:
1352N/A yield get_update(pub, stem, entry)
1352N/A return
1352N/A
2810N/A def validate(self, signatures=None, require_signatures=False):
1352N/A """Verifies whether the signatures for the contents of the
1352N/A CatalogUpdate match the specified signature data, or if not
1352N/A provided, the current signature data. Raises the exception
1352N/A named 'BadCatalogSignatures' on failure."""
1352N/A
2810N/A if not self.signatures and not signatures and \
2810N/A not require_signatures:
2810N/A # Nothing to validate, and we're not required to.
1352N/A return
1352N/A
2022N/A # Ensure content is loaded before attempting to retrieve
2022N/A # or generate signature data.
2022N/A self.load()
1352N/A if not signatures:
1352N/A signatures = self.signatures
1352N/A
1352N/A new_signatures = self._gen_signatures(self.__data)
1352N/A if new_signatures != signatures:
1352N/A raise api_errors.BadCatalogSignatures(self.pathname)
1352N/A
1352N/A
1352N/Aclass CatalogAttrs(CatalogPartBase):
1352N/A """A CatalogAttrs object is the representation of the attributes of a
1352N/A Catalog object."""
1352N/A
1352N/A # Properties.
1352N/A __data = None
1352N/A
2810N/A # This structure defines defaults (for use in __init__) as well as
2810N/A # the set of required elements for this catalog part. See also the
2810N/A # logic in load().
2810N/A __DEFAULT_ELEMS = {
2810N/A "created": None,
2810N/A "last-modified": None,
2810N/A "package-count": 0,
2810N/A "package-version-count": 0,
2810N/A "parts": {},
2810N/A "updates": {},
2810N/A "version": 1,
2810N/A }
2810N/A
3362N/A def __init__(self, meta_root=None, sign=True, file_root=None):
1352N/A """Initializes a CatalogAttrs object."""
1352N/A
1352N/A self.__data = {}
1352N/A CatalogPartBase.__init__(self, name="catalog.attrs",
3362N/A meta_root=meta_root, sign=sign, file_root=file_root)
814N/A
1352N/A if self.loaded:
1352N/A # If the data is already seen as 'loaded' during init,
1352N/A # this is actually a new object, so setup some sane
1352N/A # defaults.
1352N/A created = self.__data["last-modified"]
2810N/A self.__data = copy.deepcopy(self.__DEFAULT_ELEMS)
2810N/A self.__data["created"] = created
2810N/A self.__data["last-modified"] = created
1352N/A else:
1352N/A # Assume that the attributes of the catalog can be
1352N/A # obtained from a file.
1352N/A self.load()
1352N/A
1352N/A def __get_created(self):
1352N/A return self.__data["created"]
1352N/A
1352N/A def __get_last_modified(self):
1352N/A return self.__data["last-modified"]
1352N/A
1352N/A def __get_package_count(self):
1352N/A return self.__data["package-count"]
1352N/A
1352N/A def __get_package_version_count(self):
1352N/A return self.__data["package-version-count"]
1352N/A
1352N/A def __get_parts(self):
1352N/A return self.__data["parts"]
1352N/A
1352N/A def __get_updates(self):
1352N/A return self.__data["updates"]
1352N/A
1352N/A def __get_version(self):
1352N/A return self.__data["version"]
1352N/A
1352N/A def __set_created(self, value):
1352N/A self.__data["created"] = value
1352N/A self.signatures = {}
1352N/A
1352N/A def __set_last_modified(self, value):
1352N/A self.__data["last-modified"] = value
1352N/A self.signatures = {}
1352N/A
1352N/A def __set_package_count(self, value):
1352N/A self.__data["package-count"] = value
1352N/A self.signatures = {}
1352N/A
1352N/A def __set_package_version_count(self, value):
1352N/A self.__data["package-version-count"] = value
1352N/A self.signatures = {}
1352N/A
1352N/A def __set_parts(self, value):
1352N/A self.__data["parts"] = value
1352N/A self.signatures = {}
1352N/A
1352N/A def __set_updates(self, value):
1352N/A self.__data["updates"] = value
1352N/A self.signatures = {}
1352N/A
1352N/A def __set_version(self, value):
1352N/A self.__data["version"] = value
1352N/A self.signatures = {}
1352N/A
1352N/A def __transform(self):
1352N/A """Duplicate and transform 'self.__data' for saving."""
1352N/A
1352N/A # Use a copy to prevent the in-memory version from being
1352N/A # affected by the transformations.
1352N/A struct = copy.deepcopy(self.__data)
3234N/A for key, val in six.iteritems(struct):
1352N/A if isinstance(val, datetime.datetime):
1352N/A # Convert datetime objects to an ISO-8601
1352N/A # basic format string.
1352N/A struct[key] = datetime_to_basic_ts(val)
1352N/A continue
1352N/A
1352N/A if key in ("parts", "updates"):
1352N/A for e in val:
1352N/A lm = val[e].get("last-modified", None)
1352N/A if lm:
1352N/A lm = datetime_to_basic_ts(lm)
1352N/A val[e]["last-modified"] = lm
1352N/A return struct
1352N/A
1352N/A def load(self):
1352N/A """Load and transform the catalog attribute data."""
1352N/A
1352N/A if self.loaded:
1352N/A # Already loaded, or only in-memory.
1352N/A return
2810N/A location = os.path.join(self.meta_root, self.name)
1352N/A
1352N/A struct = CatalogPartBase.load(self)
2810N/A # Check to see that struct is as we expect: it must be a dict
2810N/A # and have all of the elements in self.__DEFAULT_ELEMS.
3194N/A if type(struct) != dict or \
2810N/A not (set(self.__DEFAULT_ELEMS.keys()) <= \
2810N/A set(struct.keys())):
2810N/A raise api_errors.InvalidCatalogFile(location)
2810N/A
2810N/A def cat_ts_to_datetime(val):
2810N/A try:
2810N/A return basic_ts_to_datetime(val)
2810N/A except ValueError:
2810N/A raise api_errors.InvalidCatalogFile(location)
2810N/A
3234N/A for key, val in six.iteritems(struct):
1352N/A if key in ("created", "last-modified"):
1352N/A # Convert ISO-8601 basic format strings to
3329N/A # datetime objects.
1352N/A if val:
2810N/A struct[key] = cat_ts_to_datetime(val)
1352N/A continue
1352N/A
1352N/A if key in ("parts", "updates"):
3194N/A if type(val) != dict:
2810N/A raise api_errors.InvalidCatalogFile(
2810N/A location)
2810N/A
2810N/A # 'parts' and 'updates' have a more complex
2810N/A # structure. Check that all of the subparts
2810N/A # look sane.
2810N/A for subpart in val:
2810N/A if subpart != os.path.basename(subpart):
2810N/A raise api_errors.\
2810N/A UnrecognizedCatalogPart(
3158N/A "{0} {{{1}: {2}}}".format(
3158N/A self.name, key, subpart))
3362N/A # Check if subpart is a symbolic link
3362N/A # that would cause an access to be
3362N/A # redirected outside of 'file_root'.
3362N/A try:
3362N/A misc.open_image_file(
3362N/A self.file_root,
3362N/A os.path.join(self.meta_root,
3362N/A subpart), os.O_RDONLY,
3362N/A misc.PKG_FILE_MODE)
3362N/A except OSError as e:
3362N/A if e.errno == errno.EREMOTE:
3362N/A raise api_errors.\
3362N/A UnrecognizedCatalogPart(
3362N/A "{0} {{{1}: {2}}}".format(
3362N/A self.name, key, subpart))
2810N/A
2810N/A # Build datetimes from timestamps.
1352N/A for e in val:
1352N/A lm = val[e].get("last-modified", None)
1352N/A if lm:
2810N/A lm = cat_ts_to_datetime(lm)
1352N/A val[e]["last-modified"] = lm
2810N/A
1352N/A self.__data = struct
1352N/A
1352N/A def save(self):
1352N/A """Transform and store the catalog attribute data in a file
1352N/A using the pathname <self.meta_root>/<self.name>."""
1352N/A
1352N/A if not self.meta_root:
1352N/A # Assume this is in-memory only.
1352N/A return
2022N/A
2022N/A # Ensure content is loaded before attempting save.
2022N/A self.load()
2022N/A
1833N/A CatalogPartBase.save(self, self.__transform(), single_pass=True)
1352N/A
2810N/A def validate(self, signatures=None, require_signatures=False):
1352N/A """Verifies whether the signatures for the contents of the
1352N/A CatalogAttrs match the specified signature data, or if not
1352N/A provided, the current signature data. Raises the exception
1352N/A named 'BadCatalogSignatures' on failure."""
1352N/A
2810N/A if not self.signatures and not signatures and \
2810N/A not require_signatures:
2810N/A # Nothing to validate, and we're not required to.
1352N/A return
1352N/A
2022N/A # Ensure content is loaded before attempting to retrieve
2022N/A # or generate signature data.
2022N/A self.load()
1352N/A if not signatures:
1352N/A signatures = self.signatures
1352N/A
1352N/A new_signatures = self._gen_signatures(self.__transform())
1352N/A if new_signatures != signatures:
1352N/A raise api_errors.BadCatalogSignatures(self.pathname)
1352N/A
1352N/A created = property(__get_created, __set_created)
1352N/A
1352N/A last_modified = property(__get_last_modified, __set_last_modified)
1352N/A
1352N/A package_count = property(__get_package_count, __set_package_count)
1352N/A
1352N/A package_version_count = property(__get_package_version_count,
1352N/A __set_package_version_count)
1352N/A
1352N/A parts = property(__get_parts, __set_parts)
1352N/A
1352N/A updates = property(__get_updates, __set_updates)
1352N/A
1352N/A version = property(__get_version, __set_version)
814N/A
852N/A
34N/Aclass Catalog(object):
1352N/A """A Catalog is the representation of the package FMRIs available from
1352N/A a package repository."""
37N/A
1431N/A __BASE_PART = "catalog.base.C"
1431N/A __DEPS_PART = "catalog.dependency.C"
1431N/A __SUMM_PART_PFX = "catalog.summary"
1352N/A
1352N/A # The file mode to be used for all catalog files.
1352N/A __file_mode = stat.S_IRUSR|stat.S_IWUSR|stat.S_IRGRP|stat.S_IROTH
1352N/A
1352N/A # These properties are declared here so that they show up in the pydoc
1352N/A # documentation as private, and for clarity in the property declarations
1352N/A # found near the end of the class definition.
1352N/A _attrs = None
1352N/A __batch_mode = None
1352N/A __lock = None
1352N/A __meta_root = None
3362N/A __file_root = None
1358N/A __sign = None
23N/A
1352N/A # These are used to cache or store CatalogPart and CatalogUpdate objects
1352N/A # as they are used. It should not be confused with the CatalogPart
1352N/A # names and CatalogUpdate names stored in the CatalogAttrs object.
1352N/A __parts = None
1352N/A __updates = None
1352N/A
1352N/A # Class Constants
1369N/A DEPENDENCY, SUMMARY = range(2)
1352N/A
1352N/A def __init__(self, batch_mode=False, meta_root=None, log_updates=False,
3362N/A read_only=False, sign=True, file_root=None):
1352N/A """Initializes a Catalog object.
157N/A
1352N/A 'batch_mode' is an optional boolean value that indicates that
1352N/A the caller intends to perform multiple modifying operations on
1352N/A catalog before saving. This is useful for performance reasons
1352N/A as the contents of the catalog will not be sorted after each
1352N/A change, and the package counts will not be updated (except at
1352N/A save()). By default this value is False. If this value is
1352N/A True, callers are responsible for calling finalize() to ensure
1352N/A that catalog entries are in the correct order and package counts
1352N/A accurately reflect the catalog contents.
429N/A
1352N/A 'meta_root' is an optional absolute pathname of a directory
1352N/A that catalog metadata can be written to and read from, and
1352N/A must already exist. If no path is supplied, then it is
1352N/A assumed that the catalog object will be used for in-memory
1352N/A operations only.
445N/A
1352N/A 'log_updates' is an optional boolean value indicating whether
1352N/A updates to the catalog should be logged. This enables consumers
1352N/A of the catalog to perform incremental updates.
34N/A
1352N/A 'read_only' is an optional boolean value that indicates if
1352N/A operations that modify the catalog are allowed (an assertion
1358N/A error will be raised if one is attempted and this is True).
1358N/A
1358N/A 'sign' is an optional boolean value that indicates that the
1358N/A the catalog data should have signature data generated and
1358N/A embedded when serialized. This option is primarily a matter
1358N/A of convenience for callers that wish to trade integrity checks
1358N/A for improved catalog serialization performance."""
1352N/A
1352N/A self.__batch_mode = batch_mode
1352N/A self.__parts = {}
1352N/A self.__updates = {}
157N/A
1352N/A # Must be set after the above.
1431N/A self.log_updates = log_updates
1352N/A self.meta_root = meta_root
1352N/A self.read_only = read_only
1358N/A self.sign = sign
3362N/A self.file_root = file_root
3362N/A if not self.file_root:
3362N/A self.file_root = meta_root
1352N/A
1352N/A # Must be set after the above.
3362N/A self._attrs = CatalogAttrs(meta_root=self.meta_root, sign=sign,
3362N/A file_root=file_root)
157N/A
1431N/A # This lock is used to protect the catalog file from multiple
1431N/A # threads writing to it at the same time.
1431N/A self.__lock = threading.Lock()
157N/A
1352N/A # Must be done last.
814N/A self.__set_perms()
814N/A
1537N/A def __actions(self, info_needed, excludes=EmptyI, cb=None, locales=None,
1537N/A last_version=False, ordered=False, pubs=EmptyI):
1537N/A assert info_needed
1537N/A if not locales:
1537N/A locales = set(("C",))
1537N/A else:
1537N/A locales = set(locales)
1537N/A
1537N/A for f, entry in self.__entries(cb=cb, info_needed=info_needed,
1537N/A locales=locales, last_version=last_version,
1537N/A ordered=ordered, pubs=pubs):
2639N/A try:
1970N/A yield f, self.__gen_actions(f, entry["actions"],
1537N/A excludes)
2639N/A except KeyError:
3329N/A yield f, EmptyI
1537N/A
1463N/A def __append(self, src, cb=None, pfmri=None, pubs=EmptyI):
1463N/A """Private version; caller responsible for locking."""
1463N/A
1463N/A base = self.get_part(self.__BASE_PART)
1463N/A src_base = src.get_part(self.__BASE_PART, must_exist=True)
1537N/A if src_base is None:
1463N/A if pfmri:
1463N/A raise api_errors.UnknownCatalogEntry(pfmri)
1463N/A # Nothing to do
1463N/A return
1463N/A
1463N/A # Use the same operation time and date for all operations so
1463N/A # that the last modification times will be synchronized. This
1463N/A # also has the benefit of avoiding extra datetime object
1463N/A # instantiations.
1463N/A op_time = datetime.datetime.utcnow()
1463N/A
1463N/A # For each entry in the 'src' catalog, add its BASE entry to the
1463N/A # current catalog along and then add it to the 'd'iscard dict if
1463N/A # 'cb' is defined and returns False.
1463N/A if pfmri:
1537N/A entry = src_base.get_entry(pfmri)
1537N/A if entry is None:
1537N/A raise api_errors.UnknownCatalogEntry(
1537N/A pfmri.get_fmri())
1537N/A entries = [(pfmri, entry)]
1463N/A else:
1463N/A entries = src_base.entries()
1463N/A
1463N/A d = {}
1463N/A for f, entry in entries:
1463N/A if pubs and f.publisher not in pubs:
1463N/A continue
1463N/A
1463N/A nentry = copy.deepcopy(entry)
1537N/A if cb is not None:
1463N/A merge, mdata = cb(src, f, entry)
1463N/A if not merge:
1463N/A pub = d.setdefault(f.publisher, {})
1463N/A plist = pub.setdefault(f.pkg_name,
1463N/A set())
1463N/A plist.add(f.version)
1463N/A continue
1463N/A
1463N/A if mdata:
1463N/A if "metadata" in nentry:
1463N/A nentry["metadata"].update(mdata)
1463N/A else:
1463N/A nentry["metadata"] = mdata
1463N/A base.add(f, metadata=nentry, op_time=op_time)
1463N/A
1463N/A if d and pfmri:
1463N/A # If the 'd'iscards dict is populated and pfmri is
1463N/A # defined, then there is nothing more to do.
1463N/A return
1463N/A
1463N/A # Finally, merge any catalog part entries that exist unless the
1463N/A # FMRI is found in the 'd'iscard dict.
1463N/A for name in src.parts.keys():
1463N/A if name == self.__BASE_PART:
1463N/A continue
1463N/A
1463N/A part = src.get_part(name, must_exist=True)
1537N/A if part is None:
1463N/A # Part doesn't exist in-memory or on-disk, so
1463N/A # skip it.
1463N/A continue
1463N/A
1463N/A if pfmri:
1537N/A entry = part.get_entry(pfmri)
1537N/A if entry is None:
1463N/A # Package isn't in this part; skip it.
1463N/A continue
1537N/A entries = [(pfmri, entry)]
1463N/A else:
1463N/A entries = part.entries()
1463N/A
1463N/A npart = self.get_part(name)
1463N/A for f, entry in entries:
1463N/A if pubs and f.publisher not in pubs:
1463N/A continue
1463N/A if f.publisher in d and \
1463N/A f.pkg_name in d[f.publisher] and \
1463N/A f.version in d[f.publisher][f.pkg_name]:
1463N/A # Skip this package.
1463N/A continue
1463N/A
1463N/A nentry = copy.deepcopy(entry)
1463N/A npart.add(f, metadata=nentry, op_time=op_time)
1463N/A
1537N/A def __entries(self, cb=None, info_needed=EmptyI,
1537N/A last_version=False, locales=None, ordered=False, pubs=EmptyI,
1537N/A tuples=False):
1537N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1537N/A # Catalog contains nothing.
1537N/A return
1537N/A
1537N/A if not locales:
1537N/A locales = set(("C",))
1537N/A else:
1537N/A locales = set(locales)
1537N/A
1537N/A parts = []
1537N/A if self.DEPENDENCY in info_needed:
1537N/A part = self.get_part(self.__DEPS_PART, must_exist=True)
1537N/A if part is not None:
1537N/A parts.append(part)
1537N/A
1537N/A if self.SUMMARY in info_needed:
1537N/A for locale in locales:
1537N/A part = self.get_part(
3158N/A "{0}.{1}".format(self.__SUMM_PART_PFX,
3158N/A locale), must_exist=True)
1537N/A if part is None:
1537N/A # Data not available for this
1537N/A # locale.
1537N/A continue
1537N/A parts.append(part)
1537N/A
1537N/A def merge_entry(src, dest):
3234N/A for k, v in six.iteritems(src):
1537N/A if k == "actions":
1537N/A dest.setdefault(k, [])
1537N/A dest[k] += v
1537N/A elif k != "version":
1537N/A dest[k] = v
1537N/A
1537N/A if tuples:
1537N/A for r, bentry in base.tuple_entries(cb=cb,
1537N/A last=last_version, ordered=ordered, pubs=pubs):
1537N/A pub, stem, ver = r
1537N/A mdata = {}
1537N/A merge_entry(bentry, mdata)
1537N/A for part in parts:
1537N/A entry = part.get_entry(pub=pub,
1537N/A stem=stem, ver=ver)
1537N/A if entry is None:
1537N/A # Part doesn't have this FMRI,
1537N/A # so skip it.
1537N/A continue
3234N/A for k, v in six.iteritems(entry):
1537N/A if k == "actions":
1537N/A mdata.setdefault(k, [])
1537N/A mdata[k] += v
1537N/A elif k != "version":
1537N/A mdata[k] = v
1537N/A yield r, mdata
1537N/A return
1537N/A
1537N/A for f, bentry in base.entries(cb=cb, last=last_version,
1537N/A ordered=ordered, pubs=pubs):
1537N/A mdata = {}
1537N/A merge_entry(bentry, mdata)
1537N/A for part in parts:
1537N/A entry = part.get_entry(f)
1537N/A if entry is None:
1537N/A # Part doesn't have this FMRI,
1537N/A # so skip it.
1537N/A continue
3234N/A for k, v in six.iteritems(entry):
1537N/A if k == "actions":
1537N/A mdata.setdefault(k, [])
1537N/A mdata[k] += v
1537N/A elif k != "version":
1537N/A mdata[k] = v
1537N/A yield f, mdata
1537N/A
1463N/A def __finalize(self, pfmris=None, pubs=None, sort=True):
1352N/A """Private finalize method; exposes additional controls for
1352N/A internal callers."""
1352N/A
1352N/A package_count = 0
1352N/A package_version_count = 0
1352N/A
1431N/A part = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if part is not None:
1352N/A # If the base Catalog didn't exist (in-memory or on-
1352N/A # disk) that implies there is nothing to sort and
1352N/A # there are no packages (since the base catalog part
1352N/A # must always exist for packages to be present).
1352N/A package_count, package_version_count = \
1352N/A part.get_package_counts()
1352N/A
1352N/A if sort:
1352N/A # Some operations don't need this, such as
1352N/A # remove...
1352N/A for part in self.__parts.values():
1463N/A part.sort(pfmris=pfmris, pubs=pubs)
1352N/A
1352N/A self._attrs.package_count = package_count
1352N/A self._attrs.package_version_count = \
1352N/A package_version_count
1352N/A
1369N/A @staticmethod
1970N/A def __gen_actions(pfmri, actions, excludes=EmptyI):
1970N/A errors = None
3110N/A if not isinstance(pfmri, fmri.PkgFmri):
3110N/A # pfmri is assumed to be a FMRI tuple.
3110N/A pub, stem, ver = pfmri
3110N/A else:
3110N/A pub = pfmri.publisher
1369N/A for astr in actions:
1970N/A try:
1970N/A a = pkg.actions.fromstr(astr)
3171N/A except pkg.actions.ActionError as e:
1970N/A # Accumulate errors and continue so that as
1970N/A # much of the action data as possible can be
1970N/A # parsed.
1970N/A if errors is None:
1970N/A # Allocate this here to avoid overhead
1970N/A # of list allocation/deallocation.
1970N/A errors = []
1970N/A if not isinstance(pfmri, fmri.PkgFmri):
2639N/A pfmri = fmri.PkgFmri(name=stem,
2639N/A publisher=pub, version=ver)
1970N/A e.fmri = pfmri
1970N/A errors.append(e)
1970N/A continue
1970N/A
1537N/A if a.name == "set" and \
1537N/A (a.attrs["name"].startswith("facet") or
1537N/A a.attrs["name"].startswith("variant")):
1537N/A # Don't filter actual facet or variant
1537N/A # set actions.
1537N/A yield a
3110N/A elif a.include_this(excludes,
3110N/A publisher=pub):
1369N/A yield a
1369N/A
1970N/A if errors is not None:
1970N/A raise api_errors.InvalidPackageErrors(errors)
1970N/A
1369N/A @staticmethod
1369N/A def __gen_manifest_actions(m, atypes, excludes):
1369N/A """Private helper function to iterate over a Manifest's actions
1369N/A by action type, returning tuples of (action, attr_name)."""
3110N/A pub = m.publisher
1369N/A for atype in atypes:
1369N/A for a in m.gen_actions_by_type(atype):
3110N/A if not a.include_this(excludes,
3110N/A publisher=pub):
1369N/A continue
1369N/A
1369N/A if atype == "set":
1369N/A yield a, a.attrs["name"]
1369N/A else:
1369N/A yield a, None
1369N/A
1352N/A def __get_batch_mode(self):
1352N/A return self.__batch_mode
1352N/A
1369N/A def __get_last_modified(self):
1369N/A return self._attrs.last_modified
1369N/A
1352N/A def __get_meta_root(self):
1352N/A return self.__meta_root
1352N/A
3362N/A def __get_file_root(self):
3362N/A return self.__file_root
3362N/A
1358N/A def __get_sign(self):
1358N/A return self.__sign
1358N/A
1352N/A def __get_update(self, name, cache=True, must_exist=False):
1352N/A # First, check if the update has already been cached,
1352N/A # and if so, return it.
1537N/A ulog = self.__updates.get(name, None)
1537N/A if ulog is not None:
1537N/A return ulog
1537N/A elif not self.meta_root and must_exist:
1537N/A return
1352N/A
1352N/A # Next, if the update hasn't been cached,
1352N/A # create an object for it.
1431N/A ulog = CatalogUpdate(name, meta_root=self.meta_root,
1431N/A sign=self.__sign)
1352N/A if self.meta_root and must_exist and not ulog.exists:
1352N/A # Update doesn't exist on-disk,
1352N/A # so don't return anything.
1352N/A return
1352N/A if cache:
1352N/A self.__updates[name] = ulog
1352N/A return ulog
1352N/A
1369N/A def __get_version(self):
1369N/A return self._attrs.version
1369N/A
1352N/A def __lock_catalog(self):
1352N/A """Locks the catalog preventing multiple threads or external
1352N/A consumers of the catalog from modifying it during operations.
1352N/A """
1352N/A
1352N/A # XXX need filesystem lock too?
1352N/A self.__lock.acquire()
1352N/A
1352N/A def __log_update(self, pfmri, operation, op_time, entries=None):
1352N/A """Helper function to log catalog changes."""
1352N/A
1352N/A if not self.__batch_mode:
1352N/A # The catalog.attrs needs to be updated to reflect
1463N/A # the changes made. A sort doesn't need to be done
1463N/A # here as the individual parts will automatically do
1463N/A # that as needed in this case.
1463N/A self.__finalize(sort=False)
1352N/A
1352N/A # This must be set to exactly the same time as the update logs
1352N/A # so that the changes in the update logs are not marked as
1352N/A # being newer than the catalog or vice versa.
1352N/A attrs = self._attrs
1352N/A attrs.last_modified = op_time
1352N/A
1431N/A if not self.log_updates:
1352N/A return
814N/A
1352N/A updates = {}
1352N/A for pname in entries:
1352N/A # The last component of the updatelog filename is the
1352N/A # related locale.
1352N/A locale = pname.split(".", 2)[2]
1537N/A updates.setdefault(locale, {})
1537N/A parts = updates[locale]
1352N/A parts[pname] = entries[pname]
1352N/A
1352N/A logdate = datetime_to_update_ts(op_time)
3234N/A for locale, metadata in six.iteritems(updates):
3158N/A name = "update.{0}.{1}".format(logdate, locale)
1352N/A ulog = self.__get_update(name)
1352N/A ulog.add(pfmri, operation, metadata=metadata,
1352N/A op_time=op_time)
1352N/A attrs.updates[name] = {
1352N/A "last-modified": op_time
1352N/A }
1352N/A
3234N/A for name, part in six.iteritems(self.__parts):
1352N/A # Signature data for each part needs to be cleared,
1352N/A # and will only be available again after save().
1352N/A attrs.parts[name] = {
1352N/A "last-modified": part.last_modified
1352N/A }
1352N/A
2510N/A @staticmethod
2510N/A def __parse_fmri_patterns(patterns):
2510N/A """A generator function that yields a list of tuples of the form
2510N/A (pattern, error, fmri, matcher) based on the provided patterns,
2510N/A where 'error' is any exception encountered while parsing the
2510N/A pattern, 'fmri' is the resulting FMRI object, and 'matcher' is
2510N/A one of the following pkg.fmri matching functions:
2510N/A
2510N/A pkg.fmri.exact_name_match
2510N/A Indicates that the name portion of the pattern
2510N/A must match exactly and the version (if provided)
2510N/A must be considered a successor or equal to the
2510N/A target FMRI.
2510N/A
2510N/A pkg.fmri.fmri_match
2510N/A Indicates that the name portion of the pattern
2510N/A must be a proper subset and the version (if
2510N/A provided) must be considered a successor or
2510N/A equal to the target FMRI.
2510N/A
2510N/A pkg.fmri.glob_match
2510N/A Indicates that the name portion of the pattern
2510N/A uses fnmatch rules for pattern matching (shell-
2510N/A style wildcards) and that the version can either
2510N/A match exactly, match partially, or contain
2510N/A wildcards.
2510N/A """
2510N/A
2510N/A for pat in patterns:
2510N/A error = None
2510N/A matcher = None
2510N/A npat = None
2510N/A try:
2510N/A parts = pat.split("@", 1)
2510N/A pat_stem = parts[0]
2510N/A pat_ver = None
2510N/A if len(parts) > 1:
2510N/A pat_ver = parts[1]
2510N/A
2510N/A if "*" in pat_stem or "?" in pat_stem:
2510N/A matcher = fmri.glob_match
2510N/A elif pat_stem.startswith("pkg:/") or \
2510N/A pat_stem.startswith("/"):
2510N/A matcher = fmri.exact_name_match
2510N/A else:
2510N/A matcher = fmri.fmri_match
2510N/A
2510N/A if matcher == fmri.glob_match:
2958N/A npat = fmri.MatchingPkgFmri(pat_stem)
2510N/A else:
2958N/A npat = fmri.PkgFmri(pat_stem)
2510N/A
2510N/A if not pat_ver:
2510N/A # Do nothing.
2510N/A pass
2510N/A elif "*" in pat_ver or "?" in pat_ver or \
2510N/A pat_ver == "latest":
2510N/A npat.version = \
2958N/A pkg.version.MatchingVersion(pat_ver)
2510N/A else:
2510N/A npat.version = \
2958N/A pkg.version.Version(pat_ver)
2510N/A
3171N/A except (fmri.FmriError, pkg.version.VersionError) as e:
2510N/A # Whatever the error was, return it.
2510N/A error = e
2510N/A yield (pat, error, npat, matcher)
2510N/A
1352N/A def __save(self):
1352N/A """Private save function. Caller is responsible for locking
1352N/A the catalog."""
1352N/A
1352N/A attrs = self._attrs
1431N/A if self.log_updates:
3234N/A for name, ulog in six.iteritems(self.__updates):
1352N/A ulog.save()
1352N/A
1352N/A # Replace the existing signature data
1352N/A # with the new signature data.
1352N/A entry = attrs.updates[name] = {
1352N/A "last-modified": ulog.last_modified
1352N/A }
3234N/A for n, v in six.iteritems(ulog.signatures):
3158N/A entry["signature-{0}".format(n)] = v
1352N/A
1352N/A # Save any CatalogParts that are currently in-memory,
1352N/A # updating their related information in catalog.attrs
1352N/A # as they are saved.
3234N/A for name, part in six.iteritems(self.__parts):
1352N/A # Must save first so that signature data is
1352N/A # current.
1833N/A
1833N/A # single-pass encoding is not used for summary part as
2810N/A # it increases memory usage substantially (30MB at
1833N/A # current for /dev). No significant difference is
1833N/A # detectable for other parts though.
1833N/A single_pass = name in (self.__BASE_PART,
1833N/A self.__DEPS_PART)
1833N/A part.save(single_pass=single_pass)
1352N/A
1352N/A # Now replace the existing signature data with
1352N/A # the new signature data.
1352N/A entry = attrs.parts[name] = {
1352N/A "last-modified": part.last_modified
1352N/A }
3234N/A for n, v in six.iteritems(part.signatures):
3158N/A entry["signature-{0}".format(n)] = v
1352N/A
1352N/A # Finally, save the catalog attributes.
1352N/A attrs.save()
1352N/A
1352N/A def __set_batch_mode(self, value):
1352N/A self.__batch_mode = value
1352N/A for part in self.__parts.values():
1352N/A part.ordered = not self.__batch_mode
1352N/A
1369N/A def __set_last_modified(self, value):
1369N/A self._attrs.last_modified = value
1369N/A
1352N/A def __set_meta_root(self, pathname):
1352N/A if pathname:
1352N/A pathname = os.path.abspath(pathname)
1352N/A self.__meta_root = pathname
1352N/A
1352N/A # If the Catalog's meta_root changes, the meta_root of all of
1352N/A # its parts must be changed too.
1352N/A if self._attrs:
1352N/A self._attrs.meta_root = pathname
1352N/A
1352N/A for part in self.__parts.values():
1352N/A part.meta_root = pathname
1352N/A
1352N/A for ulog in self.__updates.values():
1352N/A ulog.meta_root = pathname
1352N/A
3362N/A def __set_file_root(self, pathname):
3362N/A if pathname:
3362N/A pathname = os.path.abspath(pathname)
3362N/A self.__file_root = pathname
3362N/A
3362N/A # If the Catalog's file_root changes, the file_root of all of
3362N/A # its parts must be changed too.
3362N/A if self._attrs:
3362N/A self._attrs.file_root = pathname
3362N/A
3362N/A for part in self.__parts.values():
3362N/A part.file_root = pathname
3362N/A
3362N/A for ulog in self.__updates.values():
3362N/A ulog.file_root = pathname
3362N/A
1352N/A def __set_perms(self):
1352N/A """Sets permissions on attrs and parts if not read_only and if
1352N/A the current user can do so; raises BadCatalogPermissions if the
1352N/A permissions are wrong and cannot be corrected."""
1352N/A
1352N/A if not self.meta_root:
1352N/A # Nothing to do.
1352N/A return
1352N/A
1352N/A files = [self._attrs.name]
1352N/A files.extend(self._attrs.parts.keys())
1352N/A files.extend(self._attrs.updates.keys())
814N/A
814N/A # Force file_mode, so that unprivileged users can read these.
814N/A bad_modes = []
1352N/A for name in files:
1352N/A pathname = os.path.join(self.meta_root, name)
814N/A try:
814N/A if self.read_only:
1352N/A fmode = stat.S_IMODE(os.stat(
1352N/A pathname).st_mode)
1352N/A if fmode != self.__file_mode:
1352N/A bad_modes.append((pathname,
3158N/A "{0:o}".format(
3158N/A self.__file_mode),
3158N/A "{0:o}".format(fmode)))
814N/A else:
1352N/A os.chmod(pathname, self.__file_mode)
3171N/A except EnvironmentError as e:
1352N/A # If the file doesn't exist yet, move on.
814N/A if e.errno == errno.ENOENT:
814N/A continue
814N/A
814N/A # If the mode change failed for another reason,
814N/A # check to see if we actually needed to change
814N/A # it, and if so, add it to bad_modes.
1352N/A fmode = stat.S_IMODE(os.stat(
1352N/A pathname).st_mode)
1352N/A if fmode != self.__file_mode:
1352N/A bad_modes.append((pathname,
3158N/A "{0:o}".format(self.__file_mode),
3158N/A "{0:o}".format(fmode)))
814N/A
814N/A if bad_modes:
1352N/A raise api_errors.BadCatalogPermissions(bad_modes)
50N/A
1358N/A def __set_sign(self, value):
1358N/A self.__sign = value
1358N/A
1358N/A # If the Catalog's sign property changes, the value of that
1431N/A # property for its attributes, etc. must be changed too.
1358N/A if self._attrs:
1358N/A self._attrs.sign = value
1358N/A
1358N/A for part in self.__parts.values():
1358N/A part.sign = value
1358N/A
1431N/A for ulog in self.__updates.values():
1431N/A ulog.sign = value
1431N/A
1369N/A def __set_version(self, value):
1369N/A self._attrs.version = value
1369N/A
1352N/A def __unlock_catalog(self):
1352N/A """Unlocks the catalog allowing other catalog consumers to
1352N/A modify it."""
258N/A
1352N/A # XXX need filesystem unlock too?
1352N/A self.__lock.release()
1066N/A
1537N/A def actions(self, info_needed, excludes=EmptyI, cb=None,
1537N/A last=False, locales=None, ordered=False, pubs=EmptyI):
1369N/A """A generator function that produces tuples of the format
1369N/A (fmri, actions) as it iterates over the contents of the
1369N/A catalog (where 'actions' is a generator that returns the
1431N/A Actions corresponding to the requested information).
1537N/A
1431N/A If the catalog doesn't contain any action data for the package
3329N/A entry, it will return an empty iterator.
1369N/A
1369N/A 'excludes' is a list of variants which will be used to determine
1369N/A what should be allowed by the actions generator in addition to
1537N/A what is specified by 'info_needed'.
1537N/A
1537N/A 'cb' is an optional callback function that will be executed for
1537N/A each package before its action data is retrieved. It must accept
1537N/A two arguments: 'pkg' and 'entry'. 'pkg' is an FMRI object and
1537N/A 'entry' is the dictionary structure of the catalog entry for the
1537N/A package. If the callback returns False, then the entry will not
1537N/A be included in the results. This can significantly improve
1537N/A performance by avoiding action data retrieval for results that
1537N/A will not be used.
1369N/A
1369N/A 'info_needed' is a set of one or more catalog constants
1369N/A indicating the types of catalog data that will be returned
1369N/A in 'actions' in addition to the above:
1369N/A
1369N/A DEPENDENCY
1369N/A Depend and set Actions for package obsoletion,
1369N/A renaming, variants.
1369N/A
1369N/A SUMMARY
1369N/A Any remaining set Actions not listed above, such
1369N/A as pkg.summary, pkg.description, etc.
1369N/A
1537N/A 'last' is a boolean value that indicates only the last entry
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the catalog has been saved since the last modifying
1537N/A operation, or finalize() has has been called, this will also be
1537N/A the newest version of the package.
1537N/A
1369N/A 'locales' is an optional set of locale names for which Actions
1369N/A should be returned. The default is set(('C',)) if not provided.
1369N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pfmri' is an optional FMRI to limit the returned results to.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1537N/A
1537N/A return self.__actions(info_needed, excludes=excludes,
1537N/A cb=cb, last_version=last, locales=locales, ordered=ordered,
1537N/A pubs=pubs)
1369N/A
1352N/A def add_package(self, pfmri, manifest=None, metadata=None):
1352N/A """Add a package and its related metadata to the catalog and
1352N/A its parts as needed.
1066N/A
1352N/A 'manifest' is an optional Manifest object that will be used
1352N/A to retrieve the metadata related to the package.
1066N/A
1352N/A 'metadata' is an optional dict of additional metadata to store
1352N/A with the package's BASE record."""
1066N/A
1369N/A assert not self.read_only
1369N/A
1352N/A def group_actions(actions):
1352N/A dep_acts = { "C": [] }
1352N/A # Summary actions are grouped by locale, since each
1352N/A # goes to a locale-specific catalog part.
1352N/A sum_acts = { "C": [] }
1352N/A for act in actions:
1352N/A if act.name == "depend":
1352N/A dep_acts["C"].append(str(act))
1352N/A continue
429N/A
1352N/A name = act.attrs["name"]
1352N/A if name.startswith("variant") or \
1352N/A name.startswith("facet") or \
1731N/A name.startswith("pkg.depend.") or \
1461N/A name in ("pkg.obsolete", "pkg.renamed"):
1352N/A # variant and facet data goes to the
1352N/A # dependency catalog part.
1352N/A dep_acts["C"].append(str(act))
1352N/A continue
1369N/A elif name in ("fmri", "pkg.fmri"):
1352N/A # Redundant in the case of the catalog.
1352N/A continue
419N/A
1352N/A # All other set actions go to the summary
1352N/A # catalog parts, grouped by locale. To
1352N/A # determine the locale, the set attribute's
1352N/A # name is split by ':' into its field and
1352N/A # locale components. If ':' is not present,
1352N/A # then the 'C' locale is assumed.
1352N/A comps = name.split(":")
1352N/A if len(comps) > 1:
1352N/A locale = comps[1]
1352N/A else:
1352N/A locale = "C"
1352N/A if locale not in sum_acts:
1352N/A sum_acts[locale] = []
1352N/A sum_acts[locale].append(str(act))
419N/A
1352N/A return {
1352N/A "dependency": dep_acts,
1352N/A "summary": sum_acts,
419N/A }
988N/A
1352N/A self.__lock_catalog()
1352N/A try:
1352N/A entries = {}
1352N/A # Use the same operation time and date for all
1352N/A # operations so that the last modification times
1352N/A # of all catalog parts and update logs will be
1352N/A # synchronized.
1352N/A op_time = datetime.datetime.utcnow()
1352N/A
1352N/A # Always add packages to the base catalog.
1352N/A entry = {}
1352N/A if metadata:
1352N/A entry["metadata"] = metadata
1352N/A if manifest:
3234N/A for k, v in six.iteritems(manifest.signatures):
3158N/A entry["signature-{0}".format(k)] = v
1431N/A part = self.get_part(self.__BASE_PART)
1352N/A entries[part.name] = part.add(pfmri, metadata=entry,
1352N/A op_time=op_time)
1352N/A
1352N/A if manifest:
1352N/A # Without a manifest, only the base catalog data
1352N/A # can be populated.
419N/A
1352N/A # Only dependency and set actions are currently
1352N/A # used by the remaining catalog parts.
1352N/A actions = []
1352N/A for atype in "depend", "set":
1352N/A actions += manifest.gen_actions_by_type(
1352N/A atype)
419N/A
1352N/A gacts = group_actions(actions)
1352N/A for ctype in gacts:
1352N/A for locale in gacts[ctype]:
1352N/A acts = gacts[ctype][locale]
1352N/A if not acts:
1352N/A # Catalog entries only
1352N/A # added if actions are
1352N/A # present for this
1352N/A # ctype.
1352N/A continue
419N/A
1369N/A part = self.get_part("catalog"
3158N/A ".{0}.{1}".format(ctype,
3158N/A locale))
1352N/A entry = { "actions": acts }
1352N/A entries[part.name] = part.add(
1352N/A pfmri, metadata=entry,
1352N/A op_time=op_time)
419N/A
1463N/A self.__log_update(pfmri, CatalogUpdate.ADD, op_time,
1463N/A entries=entries)
1352N/A finally:
1352N/A self.__unlock_catalog()
157N/A
1431N/A def append(self, src, cb=None, pfmri=None, pubs=EmptyI):
1369N/A """Appends the entries in the specified 'src' catalog to that
1369N/A of the current catalog. The caller is responsible for ensuring
1463N/A that no duplicates exist and must call finalize() afterwards to
1463N/A to ensure consistent catalog state. This function cannot be
1463N/A used when log_updates or read_only is enabled.
1369N/A
1369N/A 'cb' is an optional callback function that must accept src,
1369N/A an FMRI, and entry. Where 'src' is the source catalog the
1369N/A FMRI's entry is being copied from, and entry is the source
1369N/A catalog entry. It must return a tuple of the form (append,
1369N/A metadata), where 'append' is a boolean value indicating if
1369N/A the specified package should be appended, and 'metadata' is
1369N/A a dict of additional metadata to store with the package's
1369N/A BASE record.
1369N/A
1369N/A 'pfmri' is an optional FMRI of a package to append. If not
1431N/A provided, all FMRIs in the 'src' catalog will be appended.
1431N/A This filtering is applied before any provided callback.
1431N/A
1431N/A 'pubs' is an optional list of publisher prefixes to restrict
1431N/A the append operation to. FRMIs that have a publisher not in
1431N/A the list will be skipped. This filtering is applied before
1431N/A any provided callback. If not provided, no publisher
1463N/A filtering will be applied."""
1431N/A
1431N/A assert not self.log_updates and not self.read_only
1431N/A
1463N/A self.__lock_catalog()
1463N/A try:
1463N/A # Append operations are much slower if batch mode is
1463N/A # not enabled. This ensures that the current state
1463N/A # is stored and then reset on completion or failure.
1463N/A # Since append() is never used as part of the
1463N/A # publication process (log_updates == True),
1463N/A # this is safe.
1463N/A old_batch_mode = self.batch_mode
1463N/A self.batch_mode = True
1463N/A self.__append(src, cb=cb, pfmri=pfmri, pubs=pubs)
1463N/A finally:
1463N/A self.batch_mode = old_batch_mode
1463N/A self.__unlock_catalog()
1369N/A
1352N/A def apply_updates(self, path):
1352N/A """Apply any CatalogUpdates available to the catalog based on
1352N/A the list returned by get_updates_needed. The caller must
1352N/A retrieve all of the resources indicated by get_updates_needed
1352N/A and place them in the directory indicated by 'path'."""
220N/A
1352N/A if not self.meta_root:
1352N/A raise api_errors.CatalogUpdateRequirements()
220N/A
1352N/A # Used to store the original time each part was modified
1352N/A # as a basis for determining whether to apply specific
1352N/A # updates.
1352N/A old_parts = self._attrs.parts
1352N/A def apply_incremental(name):
1352N/A # Load the CatalogUpdate from the path specified.
1352N/A # (Which is why __get_update is not used.)
1352N/A ulog = CatalogUpdate(name, meta_root=path)
1352N/A for pfmri, op_type, op_time, metadata in ulog.updates():
3234N/A for pname, pdata in six.iteritems(metadata):
1369N/A part = self.get_part(pname,
1352N/A must_exist=True)
1537N/A if part is None:
1352N/A # Part doesn't exist; skip.
1352N/A continue
50N/A
1352N/A lm = old_parts[pname]["last-modified"]
1431N/A if op_time <= lm:
1352N/A # Only add updates to the part
1352N/A # that occurred after the last
1352N/A # time it was originally
1352N/A # modified.
1431N/A continue
1431N/A
1431N/A if op_type == CatalogUpdate.ADD:
1352N/A part.add(pfmri, metadata=pdata,
1352N/A op_time=op_time)
1431N/A elif op_type == CatalogUpdate.REMOVE:
1431N/A part.remove(pfmri,
1431N/A op_time=op_time)
1431N/A else:
1431N/A raise api_errors.UnknownUpdateType(
1431N/A op_type)
157N/A
1352N/A def apply_full(name):
1352N/A src = os.path.join(path, name)
1352N/A dest = os.path.join(self.meta_root, name)
1352N/A portable.copyfile(src, dest)
1352N/A
1352N/A self.__lock_catalog()
1352N/A try:
1352N/A old_batch_mode = self.batch_mode
1352N/A self.batch_mode = True
157N/A
1431N/A updates = self.get_updates_needed(path)
1431N/A if updates == None:
1431N/A # Nothing has changed, so nothing to do.
1431N/A return
1431N/A
1431N/A for name in updates:
1352N/A if name.startswith("update."):
1352N/A # The provided update is an incremental.
1352N/A apply_incremental(name)
1352N/A else:
1352N/A # The provided update is a full update.
1352N/A apply_full(name)
1352N/A
1352N/A # Next, verify that all of the updated parts have a
1352N/A # signature that matches the new catalog.attrs file.
1352N/A new_attrs = CatalogAttrs(meta_root=path)
1352N/A new_sigs = {}
3234N/A for name, mdata in six.iteritems(new_attrs.parts):
1352N/A new_sigs[name] = {}
1352N/A for key in mdata:
1352N/A if not key.startswith("signature-"):
1352N/A continue
1352N/A sig = key.split("signature-")[1]
1352N/A new_sigs[name][sig] = mdata[key]
873N/A
1352N/A # This must be done to ensure that the catalog
1352N/A # signature matches that of the source.
1352N/A self.batch_mode = old_batch_mode
1352N/A self.finalize()
1352N/A
3234N/A for name, part in six.iteritems(self.__parts):
1352N/A part.validate(signatures=new_sigs[name])
1352N/A
1352N/A # Finally, save the catalog, and then copy the new
1352N/A # catalog attributes file into place and reload it.
1352N/A self.__save()
1352N/A apply_full(self._attrs.name)
873N/A
1352N/A self._attrs = CatalogAttrs(meta_root=self.meta_root)
1352N/A self.__set_perms()
1352N/A finally:
1431N/A self.batch_mode = old_batch_mode
1352N/A self.__unlock_catalog()
873N/A
1537N/A def categories(self, excludes=EmptyI, pubs=EmptyI):
1537N/A """Returns a set of tuples of the form (scheme, category)
1537N/A containing the names of all categories in use by the last
1537N/A version of each unique package in the catalog on a per-
1537N/A publisher basis.
1537N/A
1537N/A 'excludes' is a list of variants which will be used to
1537N/A determine what category actions will be checked.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1537N/A
1537N/A acts = self.__actions([self.SUMMARY], excludes=excludes,
1537N/A last_version=True, pubs=pubs)
1537N/A return set((
1537N/A sc
1537N/A for f, acts in acts
1537N/A for a in acts
1537N/A if a.has_category_info()
1537N/A for sc in a.parse_category_info()
1537N/A ))
1537N/A
1352N/A @property
1352N/A def created(self):
1352N/A """A UTC datetime object indicating the time the catalog was
1352N/A created."""
1352N/A return self._attrs.created
1352N/A
1352N/A def destroy(self):
1352N/A """Removes any on-disk files that exist for the catalog and
1352N/A discards all content."""
204N/A
1352N/A for name in self._attrs.parts:
1369N/A part = self.get_part(name)
1352N/A part.destroy()
1352N/A
1352N/A for name in self._attrs.updates:
1369N/A ulog = self.__get_update(name, cache=False)
1352N/A ulog.destroy()
204N/A
1358N/A self._attrs = CatalogAttrs(meta_root=self.meta_root,
3362N/A sign=self.__sign, file_root=self.file_root)
1352N/A self.__parts = {}
1352N/A self.__updates = {}
1606N/A self._attrs.destroy()
1606N/A
1606N/A if not self.meta_root or not os.path.exists(self.meta_root):
1606N/A return
1606N/A
1606N/A # Finally, ensure that if there are any leftover files from
1606N/A # an interrupted destroy in the past that they are removed
1606N/A # as well.
1606N/A for fname in os.listdir(self.meta_root):
1606N/A if not fname.startswith("catalog.") and \
1606N/A not fname.startswith("update."):
1606N/A continue
1606N/A
1606N/A pname = os.path.join(self.meta_root, fname)
1606N/A if not os.path.isfile(pname):
1606N/A continue
1606N/A
1606N/A try:
1606N/A portable.remove(pname)
3171N/A except EnvironmentError as e:
1606N/A if e.errno == errno.EACCES:
1606N/A raise api_errors.PermissionsException(
1606N/A e.filename)
1606N/A if e.errno == errno.EROFS:
1606N/A raise api_errors.ReadOnlyFileSystemException(
1606N/A e.filename)
1606N/A raise
1352N/A
1537N/A def entries(self, info_needed=EmptyI, last=False, locales=None,
1537N/A ordered=False, pubs=EmptyI):
1352N/A """A generator function that produces tuples of the format
1352N/A (fmri, metadata) as it iterates over the contents of the
1352N/A catalog (where 'metadata' is a dict containing the requested
1352N/A information).
1352N/A
1352N/A 'metadata' always contains the following information at a
1352N/A minimum:
220N/A
1352N/A BASE
1352N/A 'metadata' will be populated with Manifest
1352N/A signature data, if available, using key-value
1352N/A pairs of the form 'signature-<name>': value.
220N/A
1431N/A 'info_needed' is an optional list of one or more catalog
1352N/A constants indicating the types of catalog data that will
1352N/A be returned in 'metadata' in addition to the above:
220N/A
1352N/A DEPENDENCY
1352N/A 'metadata' will contain depend and set Actions
1352N/A for package obsoletion, renaming, variants,
1352N/A and facets stored in a list under the
1352N/A key 'actions'.
220N/A
1352N/A SUMMARY
1352N/A 'metadata' will contain any remaining Actions
1352N/A not listed above, such as pkg.summary,
1352N/A pkg.description, etc. in a list under the key
1352N/A 'actions'.
1352N/A
1537N/A 'last' is a boolean value that indicates only the last entry
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the catalog has been saved since the last modifying
1537N/A operation, or finalize() has has been called, this will also be
1537N/A the newest version of the package.
1537N/A
1352N/A 'locales' is an optional set of locale names for which Actions
1352N/A should be returned. The default is set(('C',)) if not provided.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1537N/A
1537N/A return self.__entries(info_needed=info_needed,
1537N/A last_version=last, locales=locales, ordered=ordered,
1537N/A pubs=pubs)
1537N/A
1537N/A def entries_by_version(self, name, info_needed=EmptyI, locales=None,
1537N/A pubs=EmptyI):
1352N/A """A generator function that produces tuples of the format
1352N/A (version, entries) as it iterates over the contents of the
1352N/A the catalog, where entries is a list of tuples of the format
1352N/A (fmri, metadata) and metadata is a dict containing the
1352N/A requested information.
1352N/A
1352N/A 'metadata' always contains the following information at a
1352N/A minimum:
1352N/A
1352N/A BASE
1352N/A 'metadata' will be populated with Manifest
1352N/A signature data, if available, using key-value
1352N/A pairs of the form 'signature-<name>': value.
1352N/A
1431N/A 'info_needed' is an optional list of one or more catalog
1352N/A constants indicating the types of catalog data that will
1352N/A be returned in 'metadata' in addition to the above:
1352N/A
1352N/A DEPENDENCY
1352N/A 'metadata' will contain depend and set Actions
1352N/A for package obsoletion, renaming, variants,
1352N/A and facets stored in a list under the
1352N/A key 'actions'.
1352N/A
1352N/A SUMMARY
1352N/A 'metadata' will contain any remaining Actions
1352N/A not listed above, such as pkg.summary,
1352N/A pkg.description, etc. in a list under the key
1352N/A 'actions'.
1352N/A
1352N/A 'locales' is an optional set of locale names for which Actions
1352N/A should be returned. The default is set(('C',)) if not provided.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1352N/A
1431N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1352N/A # Catalog contains nothing.
220N/A return
220N/A
1352N/A if not locales:
1352N/A locales = set(("C",))
1431N/A else:
1431N/A locales = set(locales)
1352N/A
1352N/A parts = []
1352N/A if self.DEPENDENCY in info_needed:
1431N/A part = self.get_part(self.__DEPS_PART, must_exist=True)
1537N/A if part is not None:
1352N/A parts.append(part)
1352N/A
1352N/A if self.SUMMARY in info_needed:
1352N/A for locale in locales:
1369N/A part = self.get_part(
3158N/A "{0}.{1}".format(self.__SUMM_PART_PFX,
3158N/A locale), must_exist=True)
1537N/A if part is None:
1352N/A # Data not available for this
1352N/A # locale.
1352N/A continue
1352N/A parts.append(part)
220N/A
1352N/A def merge_entry(src, dest):
3234N/A for k, v in six.iteritems(src):
1352N/A if k == "actions":
1369N/A dest.setdefault(k, [])
1352N/A dest[k] += v
1352N/A elif k != "version":
1352N/A dest[k] = v
1352N/A
1537N/A for ver, entries in base.entries_by_version(name, pubs=pubs):
1352N/A nentries = []
1352N/A for f, bentry in entries:
1369N/A mdata = {}
1352N/A merge_entry(bentry, mdata)
1537N/A for part in parts:
1537N/A entry = part.get_entry(f)
1537N/A if entry is None:
1537N/A # Part doesn't have this FMRI,
1537N/A # so skip it.
1537N/A continue
1537N/A merge_entry(entry, mdata)
1352N/A nentries.append((f, mdata))
1352N/A yield ver, nentries
220N/A
1537N/A def entry_actions(self, info_needed, excludes=EmptyI, cb=None,
1537N/A last=False, locales=None, ordered=False, pubs=EmptyI):
1537N/A """A generator function that produces tuples of the format
1537N/A ((pub, stem, version), entry, actions) as it iterates over
1537N/A the contents of the catalog (where 'actions' is a generator
1537N/A that returns the Actions corresponding to the requested
1537N/A information).
1537N/A
1537N/A If the catalog doesn't contain any action data for the package
3329N/A entry, it will return an empty iterator.
1537N/A
1537N/A 'excludes' is a list of variants which will be used to determine
1537N/A what should be allowed by the actions generator in addition to
1537N/A what is specified by 'info_needed'.
1537N/A
1537N/A 'cb' is an optional callback function that will be executed for
1537N/A each package before its action data is retrieved. It must accept
1537N/A two arguments: 'pkg' and 'entry'. 'pkg' is an FMRI object and
1537N/A 'entry' is the dictionary structure of the catalog entry for the
1537N/A package. If the callback returns False, then the entry will not
1537N/A be included in the results. This can significantly improve
1537N/A performance by avoiding action data retrieval for results that
1537N/A will not be used.
1537N/A
1537N/A 'info_needed' is a set of one or more catalog constants
1537N/A indicating the types of catalog data that will be returned
1537N/A in 'actions' in addition to the above:
1537N/A
1537N/A DEPENDENCY
1537N/A Depend and set Actions for package obsoletion,
1537N/A renaming, variants.
1537N/A
1537N/A SUMMARY
1537N/A Any remaining set Actions not listed above, such
1537N/A as pkg.summary, pkg.description, etc.
1537N/A
1537N/A 'last' is a boolean value that indicates only the last entry
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the catalog has been saved since the last modifying
1537N/A operation, or finalize() has has been called, this will also be
1537N/A the newest version of the package.
1537N/A
1537N/A 'locales' is an optional set of locale names for which Actions
1537N/A should be returned. The default is set(('C',)) if not provided.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pfmri' is an optional FMRI to limit the returned results to.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1537N/A
1537N/A for r, entry in self.__entries(cb=cb, info_needed=info_needed,
1537N/A locales=locales, last_version=last, ordered=ordered,
1537N/A pubs=pubs, tuples=True):
2639N/A try:
1537N/A yield (r, entry,
1970N/A self.__gen_actions(r, entry["actions"],
1537N/A excludes))
2639N/A except KeyError:
3329N/A yield r, entry, EmptyI
1537N/A
1352N/A @property
1352N/A def exists(self):
1352N/A """A boolean value indicating whether the Catalog exists
1352N/A on-disk."""
1352N/A
1352N/A # If the Catalog attrs file exists on-disk,
1352N/A # then the catalog does.
1352N/A attrs = self._attrs
1352N/A return attrs.exists
1352N/A
1463N/A def finalize(self, pfmris=None, pubs=None):
1352N/A """This function re-sorts the contents of the Catalog so that
1352N/A version entries are in the correct order and sets the package
1463N/A counts for the Catalog based on its current contents.
1463N/A
1463N/A 'pfmris' is an optional set of FMRIs that indicate what package
1463N/A entries have been changed since this function was last called.
1463N/A It is used to optimize the finalization process.
1463N/A
1463N/A 'pubs' is an optional set of publisher prefixes that indicate
1463N/A what publisher has had package entries changed. It is used
1463N/A to optimize the finalization process. This option has no effect
1463N/A if 'pfmris' is also provided."""
1463N/A
1463N/A return self.__finalize(pfmris=pfmris, pubs=pubs)
1352N/A
1537N/A def fmris(self, last=False, objects=True, ordered=False, pubs=EmptyI):
1352N/A """A generator function that produces FMRIs as it iterates
1352N/A over the contents of the catalog.
1352N/A
1537N/A 'last' is a boolean value that indicates only the last FMRI
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the catalog has been saved since the last modifying
1537N/A operation, or finalize() has has been called, this will also be
1537N/A the newest version of the package.
1537N/A
1352N/A 'objects' is an optional boolean value indicating whether
1537N/A FMRIs should be returned as FMRI objects or as strings.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
220N/A
1431N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1352N/A # Catalog contains nothing.
1352N/A
1352N/A # This construction is necessary to get python to
1352N/A # return no results properly to callers expecting
1352N/A # a generator function.
1352N/A return iter(())
1537N/A return base.fmris(last=last, objects=objects, ordered=ordered,
1537N/A pubs=pubs)
1537N/A
1537N/A def fmris_by_version(self, name, pubs=EmptyI):
1352N/A """A generator function that produces tuples of (version,
1352N/A fmris), where fmris is a of the fmris related to the
1537N/A version, for the given package name.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1352N/A
1431N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1352N/A # Catalog contains nothing.
1352N/A
1352N/A # This construction is necessary to get python to
1352N/A # return no results properly to callers expecting
1352N/A # a generator function.
1352N/A return iter(())
1537N/A return base.fmris_by_version(name, pubs=pubs)
1352N/A
1431N/A def get_entry(self, pfmri, info_needed=EmptyI, locales=None):
1352N/A """Returns a dict containing the metadata for the specified
1537N/A FMRI containing the requested information. If the specified
1537N/A FMRI does not exist in the catalog, a value of None will be
1537N/A returned.
1352N/A
1352N/A 'metadata' always contains the following information at a
1352N/A minimum:
1352N/A
1352N/A BASE
1352N/A 'metadata' will be populated with Manifest
1352N/A signature data, if available, using key-value
1352N/A pairs of the form 'signature-<name>': value.
1352N/A
1431N/A 'info_needed' is an optional list of one or more catalog
1352N/A constants indicating the types of catalog data that will
1352N/A be returned in 'metadata' in addition to the above:
204N/A
1352N/A DEPENDENCY
1352N/A 'metadata' will contain depend and set Actions
1352N/A for package obsoletion, renaming, variants,
1352N/A and facets stored in a list under the
1352N/A key 'actions'.
1352N/A
1352N/A SUMMARY
1352N/A 'metadata' will contain any remaining Actions
1352N/A not listed above, such as pkg.summary,
1352N/A pkg.description, etc. in a list under the key
1352N/A 'actions'.
1352N/A
1352N/A 'locales' is an optional set of locale names for which Actions
1352N/A should be returned. The default is set(('C',)) if not provided.
1352N/A """
1352N/A
1352N/A def merge_entry(src, dest):
3234N/A for k, v in six.iteritems(src):
1352N/A if k == "actions":
1369N/A dest.setdefault(k, [])
1352N/A dest[k] += v
1352N/A elif k != "version":
1352N/A dest[k] = v
157N/A
1352N/A parts = []
1431N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1537N/A return
1352N/A
1352N/A if not locales:
1352N/A locales = set(("C",))
1431N/A else:
1431N/A locales = set(locales)
1352N/A
1352N/A # Always attempt to retrieve the BASE entry as FMRIs
1352N/A # must be present in the BASE catalog part.
1369N/A mdata = {}
1352N/A bentry = base.get_entry(pfmri)
1537N/A if bentry is None:
1537N/A return
1352N/A merge_entry(bentry, mdata)
204N/A
1352N/A if self.DEPENDENCY in info_needed:
1431N/A part = self.get_part(self.__DEPS_PART,
1352N/A must_exist=True)
1537N/A if part is not None:
1352N/A parts.append(part)
204N/A
1352N/A if self.SUMMARY in info_needed:
1352N/A for locale in locales:
1369N/A part = self.get_part(
3158N/A "{0}.{1}".format(self.__SUMM_PART_PFX,
3158N/A locale), must_exist=True)
1537N/A if part is None:
1352N/A # Data not available for this
1352N/A # locale.
1352N/A continue
1352N/A parts.append(part)
270N/A
1537N/A for part in parts:
1537N/A entry = part.get_entry(pfmri)
1537N/A if entry is None:
1537N/A # Part doesn't have this FMRI,
1537N/A # so skip it.
1537N/A continue
1537N/A merge_entry(entry, mdata)
1352N/A return mdata
1352N/A
1369N/A def get_entry_actions(self, pfmri, info_needed, excludes=EmptyI,
1369N/A locales=None):
1369N/A """A generator function that produces Actions as it iterates
1369N/A over the catalog entry of the specified FMRI corresponding to
1369N/A the requested information). If the catalog doesn't contain
3329N/A any action data for the package entry, it will return an empty
3329N/A iterator.
1369N/A
1369N/A 'excludes' is a list of variants which will be used to determine
1369N/A what should be allowed by the actions generator in addition to
1369N/A what is specified by 'info_needed'. If not provided, only
1369N/A 'info_needed' will determine what actions are returned.
1369N/A
1369N/A 'info_needed' is a set of one or more catalog constants
1369N/A indicating the types of catalog data that will be returned
1369N/A in 'actions' in addition to the above:
1369N/A
1369N/A DEPENDENCY
1369N/A Depend and set Actions for package obsoletion,
1369N/A renaming, variants.
1369N/A
1369N/A SUMMARY
1369N/A Any remaining set Actions not listed above, such
1369N/A as pkg.summary, pkg.description, etc.
1369N/A
1369N/A 'locales' is an optional set of locale names for which Actions
1369N/A should be returned. The default is set(('C',)) if not provided.
1369N/A """
1369N/A
1369N/A assert info_needed
1369N/A if not locales:
1369N/A locales = set(("C",))
1431N/A else:
1431N/A locales = set(locales)
1369N/A
1369N/A entry = self.get_entry(pfmri, info_needed=info_needed,
1369N/A locales=locales)
1537N/A if entry is None:
1537N/A raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1369N/A
2639N/A try:
1970N/A return self.__gen_actions(pfmri, entry["actions"],
1970N/A excludes)
2639N/A except KeyError:
3329N/A return EmptyI
1369N/A
1369N/A def get_entry_all_variants(self, pfmri):
1369N/A """A generator function that yields tuples of the format
1369N/A (var_name, variants); where var_name is the name of the
1369N/A variant and variants is a list of the variants for that
1369N/A name."""
1369N/A
1369N/A info_needed = [self.DEPENDENCY]
1369N/A entry = self.get_entry(pfmri, info_needed=info_needed)
1537N/A if entry is None:
1537N/A raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1369N/A
2639N/A try:
1970N/A actions = self.__gen_actions(pfmri, entry["actions"])
2639N/A except KeyError:
3329N/A return
1369N/A
1369N/A for a in actions:
1369N/A if a.name != "set":
1369N/A continue
1369N/A
1369N/A attr_name = a.attrs["name"]
1369N/A if not attr_name.startswith("variant"):
1369N/A continue
1369N/A yield attr_name, a.attrs["value"]
1369N/A
1431N/A def get_entry_signatures(self, pfmri):
1431N/A """A generator function that yields tuples of the form (sig,
1431N/A value) where 'sig' is the name of the signature, and 'value' is
1431N/A the raw catalog value for the signature. Please note that the
1431N/A data type of 'value' is dependent on the signature, so it may
1431N/A be a string, list, dict, etc."""
1431N/A
1431N/A entry = self.get_entry(pfmri)
1537N/A if entry is None:
1537N/A raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1431N/A return (
1431N/A (k.split("signature-")[1], v)
3234N/A for k, v in six.iteritems(entry)
1431N/A if k.startswith("signature-")
1431N/A )
1431N/A
1369N/A def get_entry_variants(self, pfmri, name):
1369N/A """A generator function that returns the variants for the
1369N/A specified variant name. If no variants exist for the
1369N/A specified name, None will be returned."""
1369N/A
1369N/A for var_name, values in self.get_entry_all_variants(pfmri):
1369N/A if var_name == name:
1369N/A # A package can only have one set of values
1369N/A # for a single variant name, so return it.
1369N/A return values
1369N/A return None
1369N/A
2510N/A def gen_packages(self, collect_attrs=False, matched=None,
2510N/A patterns=EmptyI, pubs=EmptyI, unmatched=None, return_fmris=False):
2510N/A """A generator function that produces tuples of the form:
2510N/A
2510N/A (
2510N/A (
2510N/A pub, - (string) the publisher of the package
2510N/A stem, - (string) the name of the package
2510N/A version - (string) the version of the package
2510N/A ),
2510N/A states, - (list) states
2510N/A attributes - (dict) package attributes
2510N/A )
2510N/A
2510N/A Results are always sorted by stem, publisher, and then in
2510N/A descending version order.
2510N/A
2510N/A 'collect_attrs' is an optional boolean that indicates whether
2510N/A all package attributes should be collected and returned in the
2510N/A fifth element of the return tuple. If False, that element will
2510N/A be an empty dictionary.
2510N/A
2510N/A 'matched' is an optional set to add matched patterns to.
2510N/A
2510N/A 'patterns' is an optional list of FMRI wildcard strings to
2510N/A filter results by.
2510N/A
2510N/A 'pubs' is an optional list of publisher prefixes to restrict
2510N/A the results to.
2510N/A
2510N/A 'unmatched' is an optional set to add unmatched patterns to.
2510N/A
2510N/A 'return_fmris' is an optional boolean value that indicates that
2510N/A an FMRI object should be returned in place of the (pub, stem,
2510N/A ver) tuple that is normally returned."""
2510N/A
2510N/A # Each pattern in patterns can be a partial or full FMRI, so
2510N/A # extract the individual components for use in filtering.
2510N/A newest = False
2510N/A illegals = []
2510N/A pat_tuples = {}
2510N/A latest_pats = set()
2510N/A seen = set()
2510N/A npatterns = set()
2510N/A for pat, error, pfmri, matcher in self.__parse_fmri_patterns(
2510N/A patterns):
2510N/A if error:
2510N/A illegals.append(error)
2510N/A continue
2510N/A
2510N/A # Duplicate patterns are ignored.
2510N/A sfmri = str(pfmri)
2510N/A if sfmri in seen:
2510N/A # A different form of the same pattern
2510N/A # was specified already; ignore this
2510N/A # one (e.g. pkg:/network/ping,
2510N/A # /network/ping).
2510N/A continue
2510N/A
2510N/A # Track used patterns.
2510N/A seen.add(sfmri)
2510N/A npatterns.add(pat)
2510N/A
2510N/A if getattr(pfmri.version, "match_latest", None):
2510N/A latest_pats.add(pat)
2510N/A pat_tuples[pat] = (pfmri.tuple(), matcher)
2510N/A
2510N/A patterns = npatterns
2510N/A del npatterns, seen
2510N/A
2510N/A if illegals:
2510N/A raise api_errors.PackageMatchErrors(illegal=illegals)
2510N/A
2510N/A # Keep track of listed stems for all other packages on a
2510N/A # per-publisher basis.
2510N/A nlist = collections.defaultdict(int)
2510N/A
2510N/A # Track matching patterns.
2510N/A matched_pats = set()
2510N/A pkg_matching_pats = None
2510N/A
2510N/A # Need dependency and summary actions.
2510N/A cat_info = frozenset([self.DEPENDENCY, self.SUMMARY])
2510N/A
2510N/A for t, entry, actions in self.entry_actions(cat_info,
2510N/A ordered=True, pubs=pubs):
2510N/A pub, stem, ver = t
2510N/A
2510N/A omit_ver = False
2510N/A omit_package = None
2510N/A
2510N/A pkg_stem = "!".join((pub, stem))
2510N/A if newest and pkg_stem in nlist:
2510N/A # A newer version has already been listed, so
2510N/A # any additional entries need to be marked for
2510N/A # omission before continuing.
2510N/A omit_package = True
2510N/A else:
2510N/A nlist[pkg_stem] += 1
2510N/A
2510N/A if matched is not None or unmatched is not None:
2510N/A pkg_matching_pats = set()
2510N/A if not omit_package:
2510N/A ever = None
2510N/A for pat in patterns:
2510N/A (pat_pub, pat_stem, pat_ver), matcher = \
2510N/A pat_tuples[pat]
2510N/A
2510N/A if pat_pub is not None and \
2510N/A pub != pat_pub:
2510N/A # Publisher doesn't match.
2510N/A if omit_package is None:
2510N/A omit_package = True
2510N/A continue
2510N/A
2510N/A if matcher == fmri.exact_name_match:
2510N/A if pat_stem != stem:
2510N/A # Stem doesn't match.
2510N/A if omit_package is None:
2510N/A omit_package = \
2510N/A True
2510N/A continue
2510N/A elif matcher == fmri.fmri_match:
2510N/A if not ("/" + stem).endswith(
2510N/A "/" + pat_stem):
2510N/A # Stem doesn't match.
2510N/A if omit_package is None:
2510N/A omit_package = \
2510N/A True
2510N/A continue
2510N/A elif matcher == fmri.glob_match:
2510N/A if not fnmatch.fnmatchcase(stem,
2510N/A pat_stem):
2510N/A # Stem doesn't match.
2510N/A if omit_package is None:
2510N/A omit_package = \
2510N/A True
2510N/A continue
2510N/A
2510N/A if pat_ver is not None:
2510N/A if ever is None:
2510N/A # Avoid constructing a
2510N/A # version object more
2510N/A # than once for each
2510N/A # entry.
2958N/A ever = pkg.version.Version(ver)
2510N/A if not ever.is_successor(pat_ver,
2510N/A pkg.version.CONSTRAINT_AUTO):
2510N/A if omit_package is None:
2510N/A omit_package = \
2510N/A True
2510N/A omit_ver = True
2510N/A continue
2510N/A
2510N/A if pat in latest_pats and \
2510N/A nlist[pkg_stem] > 1:
2510N/A # Package allowed by pattern,
2510N/A # but isn't the "latest"
2510N/A # version.
2510N/A if omit_package is None:
2510N/A omit_package = True
2510N/A omit_ver = True
2510N/A continue
2510N/A
2510N/A # If this entry matched at least one
2510N/A # pattern, then ensure it is returned.
2510N/A omit_package = False
2510N/A if (matched is None and
2510N/A unmatched is None):
2510N/A # It's faster to stop as soon
2510N/A # as a match is found.
2510N/A break
2510N/A
2510N/A # If caller has requested other match
2510N/A # cases be returned, then all patterns
2510N/A # must be tested for every entry. This
2510N/A # is slower, so only done if necessary.
2510N/A pkg_matching_pats.add(pat)
2510N/A
2510N/A if omit_package:
2510N/A # Package didn't match critera; skip it.
2510N/A continue
2510N/A
2510N/A # Collect attribute data if requested.
2510N/A summ = None
2510N/A
2510N/A omit_var = False
2510N/A states = set()
2510N/A if collect_attrs:
3339N/A # use OrderedDict to get a deterministic output
3398N/A attrs = collections.defaultdict(
3398N/A lambda: OrderedDict([]))
2510N/A else:
2510N/A attrs = EmptyDict
2510N/A
2510N/A try:
2510N/A for a in actions:
2510N/A if a.name != "set":
2510N/A continue
2510N/A
2510N/A atname = a.attrs["name"]
2510N/A atvalue = a.attrs["value"]
2510N/A if collect_attrs:
2510N/A atvlist = a.attrlist("value")
3398N/A # mods = frozenset(
3398N/A # (k1, frozenset([k1_1, k1_2]))
3398N/A # (k2, frozenset([k2_1, k2_2]))
3398N/A # )
3398N/A # will later be converted by the
3398N/A # caller into a dict like:
3398N/A # {
3398N/A # k1: frozenset([k1_1, k1_2]),
3398N/A # k2: frozenset([k2_1, k2_2])
3398N/A # }
2510N/A mods = frozenset(
2510N/A (k, frozenset(a.attrlist(k)))
3234N/A for k in six.iterkeys(a.attrs)
2510N/A if k not in ("name", "value")
2510N/A )
3339N/A if mods not in attrs[atname]:
3339N/A attrs[atname][mods] = atvlist
3339N/A else:
3339N/A attrs[atname][mods].extend(
3339N/A atvlist)
2510N/A
2510N/A if atname == "pkg.summary":
2510N/A summ = atvalue
2510N/A continue
2510N/A
2510N/A if atname == "description":
3398N/A if summ is not None:
3398N/A continue
3398N/A
3398N/A # Historical summary field.
3398N/A summ = atvalue
3398N/A if collect_attrs:
3398N/A if mods not in \
3398N/A attrs["pkg.summary"]:
3398N/A attrs["pkg.summary"]\
3398N/A [mods] = atvlist
3398N/A else:
3398N/A attrs["pkg.summary"]\
3398N/A [mods].extend(
3398N/A atvlist)
2510N/A continue
2510N/A
2510N/A if atname == "pkg.renamed":
2510N/A if atvalue == "true":
2510N/A states.add(
2616N/A pkgdefs.PKG_STATE_RENAMED)
2510N/A continue
2510N/A if atname == "pkg.obsolete":
2510N/A if atvalue == "true":
2510N/A states.add(
2616N/A pkgdefs.PKG_STATE_OBSOLETE)
2510N/A continue
2510N/A except api_errors.InvalidPackageErrors:
2510N/A # Ignore errors for packages that have invalid
2510N/A # or unsupported metadata.
2616N/A states.add(pkgdefs.PKG_STATE_UNSUPPORTED)
2510N/A
2510N/A if omit_package:
2510N/A # Package didn't match criteria; skip it.
2510N/A if omit_ver and nlist[pkg_stem] == 1:
2510N/A del nlist[pkg_stem]
2510N/A continue
2510N/A
2510N/A if matched is not None or unmatched is not None:
2510N/A # Only after all other filtering has been
2510N/A # applied are the patterns that the package
2510N/A # matched considered "matching".
2510N/A matched_pats.update(pkg_matching_pats)
2510N/A
2510N/A # Return the requested package data.
2510N/A if return_fmris:
2958N/A pfmri = fmri.PkgFmri(name=stem, publisher=pub,
2958N/A version=ver)
2510N/A yield (pfmri, states, attrs)
2510N/A else:
2510N/A yield (t, states, attrs)
2510N/A
2510N/A if matched is not None:
2510N/A # Caller has requested that matched patterns be
2510N/A # returned.
2510N/A matched.update(matched_pats)
2510N/A if unmatched is not None:
2510N/A # Caller has requested that unmatched patterns be
2510N/A # returned.
2510N/A unmatched.update(set(pat_tuples.keys()) - matched_pats)
2510N/A
2591N/A def get_matching_fmris(self, patterns):
2301N/A """Given a user-specified list of FMRI pattern strings, return
2301N/A a tuple of ('matching', 'references', 'unmatched'), where
2301N/A matching is a dict of matching fmris, references is a dict of
2301N/A the patterns indexed by matching FMRI, and unmatched is a set of
2301N/A the patterns that did not match any FMRIs respectively:
2301N/A
2301N/A {
2301N/A pkgname: [fmri1, fmri2, ...],
2301N/A pkgname: [fmri1, fmri2, ...],
2301N/A ...
2301N/A }
2301N/A
2301N/A {
2301N/A fmri1: [pat1, pat2, ...],
2301N/A fmri2: [pat1, pat2, ...],
2301N/A ...
2301N/A }
2301N/A
2301N/A set(['unmatched1', 'unmatchedN'])
2301N/A
2301N/A 'patterns' is the list of package patterns to match.
2301N/A
2301N/A Constraint used is always AUTO as per expected UI behavior when
2301N/A determining successor versions.
2301N/A
2301N/A Note that patterns starting w/ pkg:/ require an exact match;
2301N/A patterns containing '*' will using fnmatch rules; the default
2301N/A trailing match rules are used for remaining patterns.
2301N/A
2301N/A Exactly duplicated patterns are ignored.
2301N/A
2301N/A Routine raises PackageMatchErrors if errors occur: it is
2301N/A illegal to specify multiple different patterns that match the
2301N/A same package name. Only patterns that contain wildcards are
2301N/A allowed to match multiple packages.
2301N/A """
2301N/A
2301N/A # problems we check for
2301N/A illegals = []
2301N/A unmatched = set()
2301N/A multimatch = []
2301N/A multispec = []
2301N/A pat_data = []
2301N/A wildcard_patterns = set()
2301N/A
2510N/A # Each pattern in patterns can be a partial or full FMRI, so
2510N/A # extract the individual components for use in filtering.
2301N/A latest_pats = set()
2510N/A seen = set()
2510N/A npatterns = set()
2510N/A for pat, error, pfmri, matcher in self.__parse_fmri_patterns(
2510N/A patterns):
2510N/A if error:
2510N/A illegals.append(error)
2510N/A continue
2510N/A
2510N/A # Duplicate patterns are ignored.
2510N/A sfmri = str(pfmri)
2510N/A if sfmri in seen:
2510N/A # A different form of the same pattern
2510N/A # was specified already; ignore this
2510N/A # one (e.g. pkg:/network/ping,
2510N/A # /network/ping).
2510N/A continue
2510N/A
2510N/A # Track used patterns.
2510N/A seen.add(sfmri)
2510N/A npatterns.add(pat)
2510N/A if "*" in pfmri.pkg_name or "?" in pfmri.pkg_name:
2510N/A wildcard_patterns.add(pat)
2510N/A
2510N/A if getattr(pfmri.version, "match_latest", None):
2510N/A latest_pats.add(pat)
2762N/A pat_data.append((pat, matcher, pfmri))
2510N/A
2510N/A patterns = npatterns
2510N/A del npatterns, seen
2301N/A
2301N/A if illegals:
2301N/A raise api_errors.PackageMatchErrors(illegal=illegals)
2301N/A
2301N/A # Create a dictionary of patterns, with each value being a
2301N/A # dictionary of pkg names & fmris that match that pattern.
2301N/A ret = dict(zip(patterns, [dict() for i in patterns]))
2301N/A
2301N/A for name in self.names():
2762N/A for pat, matcher, pfmri in pat_data:
2301N/A pub = pfmri.publisher
2301N/A version = pfmri.version
2301N/A if not matcher(name, pfmri.pkg_name):
2301N/A continue # name doesn't match
2762N/A for ver, entries in \
2762N/A self.entries_by_version(name):
2301N/A if version and not ver.is_successor(
2762N/A version,
2762N/A pkg.version.CONSTRAINT_AUTO):
2301N/A continue # version doesn't match
2301N/A for f, metadata in entries:
2301N/A fpub = f.publisher
2301N/A if pub and pub != fpub:
2301N/A # specified pubs
2301N/A # conflict
2762N/A continue
2301N/A ret[pat].setdefault(f.pkg_name,
2301N/A []).append(f)
2301N/A
2301N/A # Discard all but the newest version of each match.
2301N/A if latest_pats:
2301N/A # Rebuild ret based on latest version of every package.
2301N/A latest = {}
2301N/A nret = {}
2301N/A for p in patterns:
2301N/A if p not in latest_pats or not ret[p]:
2301N/A nret[p] = ret[p]
2301N/A continue
2301N/A
2301N/A nret[p] = {}
2301N/A for pkg_name in ret[p]:
2301N/A nret[p].setdefault(pkg_name, [])
2301N/A for f in ret[p][pkg_name]:
2301N/A nver = latest.get(f.pkg_name,
2301N/A None)
2301N/A if nver > f.version:
2301N/A # Not the newest.
2301N/A continue
2301N/A if nver == f.version:
2301N/A # Allow for multiple
2301N/A # FMRIs of the same
2301N/A # latest version.
2301N/A nret[p][pkg_name].append(
2301N/A f)
2301N/A continue
2301N/A
2301N/A latest[f.pkg_name] = f.version
2301N/A nret[p][pkg_name] = [f]
2301N/A
2301N/A # Assign new version of ret and discard latest list.
2301N/A ret = nret
2301N/A del latest
2301N/A
2301N/A # Determine match failures.
2301N/A matchdict = {}
2301N/A for p in patterns:
2301N/A l = len(ret[p])
2301N/A if l == 0: # no matches at all
2301N/A unmatched.add(p)
2301N/A elif l > 1 and p not in wildcard_patterns:
2301N/A # multiple matches
2301N/A multimatch.append((p, [
2301N/A ret[p][n][0].get_pkg_stem()
2301N/A for n in ret[p]
2301N/A ]))
2301N/A else:
2301N/A # single match or wildcard
2301N/A for k in ret[p].keys():
2301N/A # for each matching package name
2301N/A matchdict.setdefault(k, []).append(p)
2301N/A
2591N/A if multimatch:
2301N/A raise api_errors.PackageMatchErrors(
2591N/A multiple_matches=multimatch)
2591N/A
2591N/A # Group the matching patterns by package name and allow multiple
2591N/A # fmri matches.
2301N/A proposed_dict = {}
2301N/A for d in ret.values():
3234N/A for k, l in six.iteritems(d):
2591N/A proposed_dict.setdefault(k, []).extend(l)
2301N/A
2301N/A # construct references so that we can know which pattern
2301N/A # generated which fmris...
2301N/A references = dict([
2301N/A (f, p)
2301N/A for p in ret.keys()
2301N/A for flist in ret[p].values()
2301N/A for f in flist
2301N/A ])
2301N/A
2301N/A return proposed_dict, references, unmatched
2301N/A
3110N/A def get_package_counts_by_pub(self, pubs=EmptyI):
1968N/A """Returns a generator of tuples of the form (pub,
1968N/A package_count, package_version_count). 'pub' is the publisher
1968N/A prefix, 'package_count' is the number of unique packages for the
1968N/A publisher, and 'package_version_count' is the number of unique
1968N/A package versions for the publisher.
1968N/A """
1968N/A
1968N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1968N/A if base is None:
1968N/A # Catalog contains nothing.
1968N/A
1968N/A # This construction is necessary to get python to
1968N/A # return no results properly to callers expecting
1968N/A # a generator function.
1968N/A return iter(())
3110N/A return base.get_package_counts_by_pub(pubs=pubs)
1968N/A
1369N/A def get_part(self, name, must_exist=False):
1369N/A """Returns the CatalogPart object for the named catalog part.
1369N/A
1369N/A 'must_exist' is an optional boolean value that indicates that
1369N/A the catalog part must already exist in-memory or on-disk, if
1369N/A not a value of None will be returned."""
1369N/A
1369N/A # First, check if the part has already been cached, and if so,
1369N/A # return it.
1537N/A part = self.__parts.get(name, None)
1537N/A if part is not None:
1537N/A return part
1537N/A elif not self.meta_root and must_exist:
1537N/A return
1369N/A
1549N/A # If the caller said the part must_exist, then it must already
1549N/A # be part of the catalog attributes to be valid.
1549N/A aparts = self._attrs.parts
1549N/A if must_exist and name not in aparts:
1549N/A return
1549N/A
1549N/A # Next, since the part hasn't been cached, create an object
1549N/A # for it and add it to catalog attributes.
1369N/A part = CatalogPart(name, meta_root=self.meta_root,
3362N/A ordered=not self.__batch_mode, sign=self.__sign,
3362N/A file_root=self.file_root)
1549N/A if must_exist and self.meta_root and not part.exists:
1549N/A # This is a double-check for the client case where
1549N/A # there is a part that is known to the catalog but
1549N/A # that the client has purposefully not retrieved.
1549N/A # (Think locale specific data.)
1369N/A return
1549N/A
1369N/A self.__parts[name] = part
1387N/A
1387N/A if name not in aparts:
1387N/A # Add a new entry to the catalog attributes for this new
1387N/A # part since it didn't exist previously.
1387N/A aparts[name] = {
1387N/A "last-modified": part.last_modified
1387N/A }
1369N/A return part
1369N/A
1352N/A def get_updates_needed(self, path):
1352N/A """Returns a list of the catalog files needed to update
1352N/A the existing catalog parts, based on the contents of the
1352N/A catalog.attrs file in the directory indicated by 'path'.
1431N/A A value of None will be returned if the the catalog has
1431N/A not been modified, while an empty list will be returned
1431N/A if no catalog parts need to be updated, but the catalog
1431N/A itself has changed."""
1352N/A
1352N/A new_attrs = CatalogAttrs(meta_root=path)
1352N/A if not new_attrs.exists:
1431N/A # No updates needed (not even to attrs), so return None.
1431N/A return None
1431N/A
1431N/A old_attrs = self._attrs
1431N/A if old_attrs.created != new_attrs.created:
1431N/A # It's very likely that the catalog has been recreated
1431N/A # or this is a completely different catalog than was
1431N/A # expected. In either case, an update isn't possible.
1431N/A raise api_errors.BadCatalogUpdateIdentity(path)
1431N/A
1431N/A if new_attrs.last_modified == old_attrs.last_modified:
1431N/A # No updates needed (not even to attrs), so return None.
1431N/A return None
270N/A
1352N/A # First, verify that all of the catalog parts the client has
1352N/A # still exist. If they no longer exist, the catalog is no
1352N/A # longer valid and cannot be updated.
1352N/A parts = {}
1352N/A incremental = True
1352N/A for name in old_attrs.parts:
1352N/A if name not in new_attrs.parts:
1352N/A raise api_errors.BadCatalogUpdateIdentity(path)
221N/A
1352N/A old_lm = old_attrs.parts[name]["last-modified"]
1352N/A new_lm = new_attrs.parts[name]["last-modified"]
30N/A
1352N/A if new_lm == old_lm:
1352N/A # Part hasn't changed.
1352N/A continue
1352N/A elif new_lm < old_lm:
1352N/A raise api_errors.ObsoleteCatalogUpdate(path)
1352N/A
1352N/A # The last component of the update name is the locale.
1352N/A locale = name.split(".", 2)[2]
1352N/A
1352N/A # Now check to see if an update log is still offered for
1352N/A # the last time this catalog part was updated. If it
1352N/A # does not, then an incremental update cannot be safely
1352N/A # performed since updates may be missing.
1352N/A logdate = datetime_to_update_ts(old_lm)
3158N/A logname = "update.{0}.{1}".format(logdate, locale)
1352N/A
1352N/A if logname not in new_attrs.updates:
1352N/A incremental = False
157N/A
1431N/A parts.setdefault(locale, set())
1352N/A parts[locale].add(name)
1352N/A
1431N/A # XXX in future, add current locale to this. For now, just
1431N/A # ensure that all of the locales of parts that were changed
1431N/A # and exist on-disk are included.
1431N/A locales = set(("C",))
1431N/A locales.update(set(parts.keys()))
1431N/A
1431N/A # Now determine if there are any new parts for this locale that
1431N/A # this version of the API knows how to use that the client
1431N/A # doesn't already have.
1431N/A for name in new_attrs.parts:
1431N/A if name in parts or name in old_attrs.parts:
1431N/A continue
1431N/A
1431N/A # The last component of the name is the locale.
1431N/A locale = name.split(".", 2)[2]
1431N/A if locale not in locales:
1431N/A continue
1431N/A
1431N/A # Currently, only these parts are used by the client,
1431N/A # so only they need to be retrieved.
1431N/A if name == self.__BASE_PART or \
1431N/A name == self.__DEPS_PART or \
1431N/A name.startswith(self.__SUMM_PART_PFX):
1431N/A incremental = False
1431N/A
1431N/A # If a new part has been added for the current
1431N/A # locale, then incremental updates can't be
1431N/A # performed since updates for this locale can
1431N/A # only be applied to parts that already exist.
1431N/A parts.setdefault(locale, set())
1431N/A parts[locale].add(name)
1431N/A
1352N/A if not parts:
1431N/A # No updates needed to catalog parts on-disk, but
1431N/A # catalog has changed.
1352N/A return []
1352N/A elif not incremental:
1352N/A # Since an incremental update cannot be performed,
1352N/A # just return the updated parts for retrieval.
1352N/A updates = set()
1352N/A for locale in parts:
1352N/A updates.update(parts[locale])
1352N/A return updates
24N/A
1352N/A # Finally, determine the update logs needed based on the catalog
1352N/A # parts that need updating on a per-locale basis.
1352N/A updates = set()
1352N/A for locale in parts:
1352N/A # Determine the newest catalog part for a given locale,
1352N/A # this will be used to determine which update logs are
1352N/A # needed for an incremental update.
1352N/A last_lm = None
1352N/A for name in parts[locale]:
1431N/A if name not in old_attrs.parts:
1431N/A continue
1431N/A
1352N/A lm = old_attrs.parts[name]["last-modified"]
1352N/A if not last_lm or lm > last_lm:
1352N/A last_lm = lm
220N/A
3234N/A for name, uattrs in six.iteritems(new_attrs.updates):
1352N/A up_lm = uattrs["last-modified"]
1352N/A
1352N/A # The last component of the update name is the
1352N/A # locale.
1352N/A up_locale = name.split(".", 2)[2]
1352N/A
1352N/A if not up_locale == locale:
1352N/A # This update log doesn't apply to the
1352N/A # locale being evaluated for updates.
258N/A continue
258N/A
1352N/A if up_lm <= last_lm:
1352N/A # Older or same as newest catalog part
1352N/A # for this locale; so skip.
1352N/A continue
59N/A
1352N/A # If this updatelog was changed after the
1352N/A # newest catalog part for this locale, then
1352N/A # it is needed to update one or more catalog
1352N/A # parts for this locale.
1352N/A updates.add(name)
215N/A
1352N/A # Ensure updates are in chronological ascending order.
1352N/A return sorted(updates)
157N/A
1537N/A def names(self, pubs=EmptyI):
1352N/A """Returns a set containing the names of all the packages in
1537N/A the Catalog.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
157N/A
1431N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1352N/A # Catalog contains nothing.
1352N/A return set()
1537N/A return base.names(pubs=pubs)
157N/A
1352N/A @property
1352N/A def package_count(self):
1352N/A """The number of unique packages in the catalog."""
1352N/A return self._attrs.package_count
310N/A
1352N/A @property
1352N/A def package_version_count(self):
1352N/A """The number of unique package versions in the catalog."""
1352N/A return self._attrs.package_version_count
621N/A
1352N/A @property
1352N/A def parts(self):
1352N/A """A dict containing the list of CatalogParts that the catalog
1352N/A is composed of along with information about each part."""
157N/A
1352N/A return self._attrs.parts
157N/A
1537N/A def pkg_names(self, pubs=EmptyI):
1537N/A """A generator function that produces package tuples of the form
1537N/A (pub, stem) as it iterates over the contents of the catalog.
1537N/A
1537N/A 'pubs' is an optional list that contains the prefixes of the
1537N/A publishers to restrict the results to."""
1537N/A
1537N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1537N/A # Catalog contains nothing.
1537N/A
1537N/A # This construction is necessary to get python to
1537N/A # return no results properly to callers expecting
1537N/A # a generator function.
1537N/A return iter(())
1537N/A return base.pkg_names(pubs=pubs)
1537N/A
1431N/A def publishers(self):
1431N/A """Returns a set containing the prefixes of all the publishers
1431N/A in the Catalog."""
1431N/A
1431N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1431N/A # Catalog contains nothing.
1431N/A return set()
1431N/A return set(p for p in base.publishers())
1431N/A
1352N/A def remove_package(self, pfmri):
1352N/A """Remove a package and its metadata."""
621N/A
1369N/A assert not self.read_only
1369N/A
1352N/A self.__lock_catalog()
1352N/A try:
1352N/A # The package has to be removed from every known part.
1352N/A entries = {}
621N/A
1352N/A # Use the same operation time and date for all
1352N/A # operations so that the last modification times
1352N/A # of all catalog parts and update logs will be
1352N/A # synchronized.
1352N/A op_time = datetime.datetime.utcnow()
157N/A
1352N/A for name in self._attrs.parts:
1369N/A part = self.get_part(name)
1537N/A if part is None:
1191N/A continue
1352N/A
1537N/A pkg_entry = part.get_entry(pfmri)
1537N/A if pkg_entry is None:
1431N/A if name == self.__BASE_PART:
1431N/A # Entry should exist in at least
1431N/A # the base part.
1537N/A raise api_errors.UnknownCatalogEntry(
1537N/A pfmri.get_fmri())
1431N/A # Skip; package's presence is optional
1431N/A # in other parts.
847N/A continue
157N/A
1352N/A part.remove(pfmri, op_time=op_time)
1431N/A if self.log_updates:
1352N/A entries[part.name] = pkg_entry
1352N/A
1352N/A self.__log_update(pfmri, CatalogUpdate.REMOVE, op_time,
1352N/A entries=entries)
1352N/A finally:
1352N/A self.__unlock_catalog()
1352N/A
1352N/A def save(self):
1352N/A """Finalize current state and save to file if possible."""
621N/A
1352N/A self.__lock_catalog()
1352N/A try:
1352N/A self.__save()
1352N/A finally:
1352N/A self.__unlock_catalog()
847N/A
1352N/A @property
1352N/A def signatures(self):
1352N/A """Returns a dict of the files the catalog is composed of along
1352N/A with the last known signatures of each if they are available."""
621N/A
1352N/A attrs = self._attrs
1352N/A sigs = {
1352N/A attrs.name: attrs.signatures
1352N/A }
1066N/A
1352N/A for items in (attrs.parts, attrs.updates):
1352N/A for name in items:
1352N/A entry = sigs[name] = {}
1352N/A for k in items[name]:
1352N/A try:
1352N/A sig = k.split("signature-")[1]
1352N/A entry[sig] = items[name][k]
1352N/A except IndexError:
1352N/A # Not a signature entry.
1352N/A continue
1352N/A return sigs
1066N/A
1537N/A def tuples(self, last=False, ordered=False, pubs=EmptyI):
1537N/A """A generator function that produces FMRI tuples as it
1537N/A iterates over the contents of the catalog.
1537N/A
1537N/A 'last' is a boolean value that indicates only the last FMRI
1537N/A tuple for each package on a per-publisher basis should be
1537N/A returned. As long as the catalog has been saved since the
1537N/A last modifying operation, or finalize() has has been called,
1537N/A this will also be the newest version of the package.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1431N/A
1431N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
1431N/A # Catalog contains nothing.
1431N/A
1431N/A # This construction is necessary to get python to
1431N/A # return no results properly to callers expecting
1431N/A # a generator function.
1431N/A return iter(())
1537N/A return base.tuples(last=last, ordered=ordered, pubs=pubs)
1537N/A
1537N/A def tuple_entries(self, info_needed=EmptyI, last=False, locales=None,
1537N/A ordered=False, pubs=EmptyI):
1537N/A """A generator function that produces tuples of the format
1537N/A ((pub, stem, version), entry, actions) as it iterates over
1537N/A the contents of the catalog (where 'metadata' is a dict
1537N/A containing the requested information).
1537N/A
1537N/A 'metadata' always contains the following information at a
1537N/A minimum:
1537N/A
1537N/A BASE
1537N/A 'metadata' will be populated with Manifest
1537N/A signature data, if available, using key-value
1537N/A pairs of the form 'signature-<name>': value.
1537N/A
1537N/A 'info_needed' is an optional list of one or more catalog
1537N/A constants indicating the types of catalog data that will
1537N/A be returned in 'metadata' in addition to the above:
1537N/A
1537N/A DEPENDENCY
1537N/A 'metadata' will contain depend and set Actions
1537N/A for package obsoletion, renaming, variants,
1537N/A and facets stored in a list under the
1537N/A key 'actions'.
1537N/A
1537N/A SUMMARY
1537N/A 'metadata' will contain any remaining Actions
1537N/A not listed above, such as pkg.summary,
1537N/A pkg.description, etc. in a list under the key
1537N/A 'actions'.
1537N/A
1537N/A 'last' is a boolean value that indicates only the last entry
1537N/A for each package on a per-publisher basis should be returned.
1537N/A As long as the catalog has been saved since the last modifying
1537N/A operation, or finalize() has has been called, this will also be
1537N/A the newest version of the package.
1537N/A
1537N/A 'locales' is an optional set of locale names for which Actions
1537N/A should be returned. The default is set(('C',)) if not provided.
1537N/A
1537N/A 'ordered' is an optional boolean value that indicates that
1537N/A results should sorted by stem and then by publisher and
1537N/A be in descending version order. If False, results will be
1537N/A in a ascending version order on a per-publisher, per-stem
1537N/A basis.
1537N/A
1537N/A 'pubs' is an optional list of publisher prefixes to restrict
1537N/A the results to."""
1537N/A
1537N/A return self.__entries(info_needed=info_needed,
1537N/A locales=locales, last_version=last, ordered=ordered,
1537N/A pubs=pubs, tuples=True)
1431N/A
1352N/A @property
1352N/A def updates(self):
1352N/A """A dict containing the list of known updates for the catalog
1352N/A along with information about each update."""
1066N/A
1352N/A return self._attrs.updates
286N/A
2352N/A def update_entry(self, metadata, pfmri=None, pub=None, stem=None,
2352N/A ver=None):
1369N/A """Updates the metadata stored in a package's BASE catalog
2352N/A record for the specified package. Cannot be used when read_only
1369N/A or log_updates is enabled; should never be used with a Catalog
1369N/A intended for incremental update usage.
1369N/A
2352N/A 'metadata' must be a dict of additional metadata to store with
2352N/A the package's BASE record.
2352N/A
1369N/A 'pfmri' is the FMRI of the package to update the entry for.
1369N/A
2352N/A 'pub' is the publisher of the package.
2352N/A
2352N/A 'stem' is the stem of the package.
2352N/A
2352N/A 'ver' is the version string of the package.
2352N/A
2352N/A 'pfmri' or 'pub', 'stem', and 'ver' must be provided.
2352N/A """
2352N/A
2352N/A assert pfmri or (pub and stem and ver)
1431N/A assert not self.log_updates and not self.read_only
1431N/A
1431N/A base = self.get_part(self.__BASE_PART, must_exist=True)
1537N/A if base is None:
2352N/A if not pfmri:
2639N/A pfmri = fmri.PkgFmri(name=stem, publisher=pub,
2639N/A version=ver)
1369N/A raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1369N/A
1369N/A # get_entry returns the actual catalog entry, so updating it
1369N/A # simply requires reassignment.
2352N/A entry = base.get_entry(pfmri=pfmri, pub=pub, stem=stem, ver=ver)
1537N/A if entry is None:
2352N/A if not pfmri:
2639N/A pfmri = fmri.PkgFmri(name=stem, publisher=pub,
2639N/A version=ver)
1537N/A raise api_errors.UnknownCatalogEntry(pfmri.get_fmri())
1369N/A if metadata is None:
1369N/A if "metadata" in entry:
1369N/A del entry["metadata"]
1369N/A return
1369N/A entry["metadata"] = metadata
1369N/A
1710N/A op_time = datetime.datetime.utcnow()
1710N/A attrs = self._attrs
1710N/A attrs.last_modified = op_time
1710N/A attrs.parts[base.name] = {
1710N/A "last-modified": op_time
1710N/A }
1710N/A base.last_modified = op_time
1710N/A
2810N/A def validate(self, require_signatures=False):
1352N/A """Verifies whether the signatures for the contents of the
1352N/A catalog match the current signature data. Raises the
1352N/A exception named 'BadCatalogSignatures' on failure."""
157N/A
2810N/A self._attrs.validate(require_signatures=require_signatures)
621N/A
2022N/A def get_sigs(mdata):
2022N/A sigs = {}
2022N/A for key in mdata:
2022N/A if not key.startswith("signature-"):
2022N/A continue
2022N/A sig = key.split("signature-")[1]
2022N/A sigs[sig] = mdata[key]
2022N/A if not sigs:
2022N/A # Allow validate() to perform its own fallback
2022N/A # logic if signature data isn't available.
2022N/A return None
2022N/A return sigs
2022N/A
3234N/A for name, mdata in six.iteritems(self._attrs.parts):
2022N/A part = self.get_part(name, must_exist=True)
1537N/A if part is None:
1352N/A # Part does not exist; no validation needed.
1352N/A continue
2810N/A part.validate(signatures=get_sigs(mdata),
2810N/A require_signatures=require_signatures)
2022N/A
3234N/A for name, mdata in six.iteritems(self._attrs.updates):
2022N/A ulog = self.__get_update(name, cache=False,
2022N/A must_exist=True)
1537N/A if ulog is None:
1352N/A # Update does not exist; no validation needed.
1352N/A continue
2810N/A ulog.validate(signatures=get_sigs(mdata),
2810N/A require_signatures=require_signatures)
157N/A
1352N/A batch_mode = property(__get_batch_mode, __set_batch_mode)
1369N/A last_modified = property(__get_last_modified, __set_last_modified,
1369N/A doc="A UTC datetime object indicating the last time the catalog "
1369N/A "was modified.")
1352N/A meta_root = property(__get_meta_root, __set_meta_root)
3362N/A file_root = property(__get_file_root, __set_file_root)
1358N/A sign = property(__get_sign, __set_sign)
1369N/A version = property(__get_version, __set_version)
814N/A
1431N/A
1431N/A# Methods used by external callers
1431N/Adef verify(filename):
1431N/A """Convert the catalog part named by filename into the correct
1431N/A type of Catalog object and then call its validate method to ensure
1431N/A that is contents are self-consistent."""
1431N/A
1431N/A path, fn = os.path.split(filename)
1431N/A catobj = None
1431N/A
1431N/A if fn.startswith("catalog"):
1431N/A if fn.endswith("attrs"):
1431N/A catobj = CatalogAttrs(meta_root=path)
1431N/A else:
1431N/A catobj = CatalogPart(fn, meta_root=path)
1431N/A elif fn.startswith("update"):
1431N/A catobj = CatalogUpdate(fn, meta_root=path)
1431N/A else:
1431N/A # Unrecognized.
1537N/A raise api_errors.UnrecognizedCatalogPart(fn)
1431N/A
1431N/A # With the else case above, this should never be None.
1431N/A assert catobj
1431N/A
2810N/A catobj.validate(require_signatures=True)
1431N/A
1352N/A# Methods used by Catalog classes.
1352N/Adef datetime_to_ts(dt):
1352N/A """Take datetime object dt, and convert it to a ts in ISO-8601
1352N/A format. """
1352N/A
1352N/A return dt.isoformat()
215N/A
1352N/Adef datetime_to_basic_ts(dt):
1352N/A """Take datetime object dt, and convert it to a ts in ISO-8601
1352N/A basic format. """
45N/A
1352N/A val = dt.isoformat()
1352N/A val = val.replace("-", "")
1352N/A val = val.replace(":", "")
621N/A
1352N/A if not dt.tzname():
1352N/A # Assume UTC.
1352N/A val += "Z"
1352N/A return val
621N/A
1352N/Adef datetime_to_update_ts(dt):
1352N/A """Take datetime object dt, and convert it to a ts in ISO-8601
1352N/A basic partial format. """
621N/A
1352N/A val = dt.isoformat()
1352N/A val = val.replace("-", "")
1352N/A # Drop the minutes and seconds portion.
1352N/A val = val.rsplit(":", 2)[0]
1352N/A val = val.replace(":", "")
621N/A
1352N/A if not dt.tzname():
1352N/A # Assume UTC.
1352N/A val += "Z"
1352N/A return val
106N/A
1352N/Adef now_to_basic_ts():
1352N/A """Returns the current UTC time as timestamp in ISO-8601 basic
1352N/A format."""
1352N/A return datetime_to_basic_ts(datetime.datetime.utcnow())
220N/A
1352N/Adef now_to_update_ts():
1352N/A """Returns the current UTC time as timestamp in ISO-8601 basic
1352N/A partial format."""
1352N/A return datetime_to_update_ts(datetime.datetime.utcnow())
215N/A
215N/Adef ts_to_datetime(ts):
1352N/A """Take timestamp ts in ISO-8601 format, and convert it to a
1352N/A datetime object."""
215N/A
215N/A year = int(ts[0:4])
215N/A month = int(ts[5:7])
215N/A day = int(ts[8:10])
215N/A hour = int(ts[11:13])
382N/A minutes = int(ts[14:16])
215N/A sec = int(ts[17:19])
289N/A # usec is not in the string if 0
289N/A try:
382N/A usec = int(ts[20:26])
289N/A except ValueError:
382N/A usec = 0
1352N/A return datetime.datetime(year, month, day, hour, minutes, sec, usec)
215N/A
1352N/Adef basic_ts_to_datetime(ts):
1352N/A """Take timestamp ts in ISO-8601 basic format, and convert it to a
1352N/A datetime object."""
215N/A
1352N/A year = int(ts[0:4])
1352N/A month = int(ts[4:6])
1352N/A day = int(ts[6:8])
1352N/A hour = int(ts[9:11])
1352N/A minutes = int(ts[11:13])
1352N/A sec = int(ts[13:15])
1352N/A # usec is not in the string if 0
1352N/A try:
1352N/A usec = int(ts[16:22])
1352N/A except ValueError:
1352N/A usec = 0
1352N/A return datetime.datetime(year, month, day, hour, minutes, sec, usec)