catalog.py revision 174
23N/A#!/usr/bin/python
23N/A#
23N/A# CDDL HEADER START
23N/A#
23N/A# The contents of this file are subject to the terms of the
23N/A# Common Development and Distribution License (the "License").
23N/A# You may not use this file except in compliance with the License.
23N/A#
23N/A# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
23N/A# or http://www.opensolaris.org/os/licensing.
23N/A# See the License for the specific language governing permissions
23N/A# and limitations under the License.
23N/A#
23N/A# When distributing Covered Code, include this CDDL HEADER in each
23N/A# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
23N/A# If applicable, add the following below this CDDL HEADER, with the
23N/A# fields enclosed by brackets "[]" replaced with your own identifying
23N/A# information: Portions Copyright [yyyy] [name of copyright owner]
23N/A#
23N/A# CDDL HEADER END
23N/A#
221N/A# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
23N/A# Use is subject to license terms.
23N/A
221N/Aimport os
221N/Aimport re
221N/Aimport sha
23N/Aimport shutil
23N/Aimport time
23N/Aimport urllib
174N/Aimport tempfile
204N/Aimport errno
204N/A
204N/Aimport pkg.fmri as fmri
215N/Aimport pkg.version as version
264N/A
270N/Aclass CatalogException(Exception):
23N/A def __init__(self, args=None):
23N/A self.args = args
157N/A
204N/Aclass Catalog(object):
204N/A """A Catalog is the representation of the package FMRIs available to
157N/A this client or repository. Both purposes utilize the same storage
157N/A format.
157N/A
157N/A The serialized structure of the repository is an unordered list of
23N/A available package versions, followed by an unordered list of
258N/A incorporation relationships between packages. This latter section
258N/A allows the graph to be topologically sorted by the client.
258N/A
258N/A S Last-Modified: [timespec]
34N/A
34N/A XXX A authority mirror-uri ...
34N/A XXX ...
34N/A
24N/A V fmri
26N/A V fmri
26N/A ...
26N/A C fmri
26N/A C fmri
26N/A ...
50N/A I fmri fmri
50N/A I fmri fmri
37N/A ...
37N/A """
37N/A
26N/A # XXX Mirroring records also need to be allowed from client
26N/A # configuration, and not just catalogs.
26N/A #
50N/A # XXX It would be nice to include available tags and package sizes,
50N/A # although this could also be calculated from the set of manifests.
50N/A #
26N/A # XXX Current code is O(N_packages) O(M_versions), should be
26N/A # O(1) O(M_versions), and possibly O(1) O(1).
26N/A #
270N/A # XXX Initial estimates suggest that the Catalog could be composed of
270N/A # 1e5 - 1e7 lines. Catalogs across these magnitudes will need to be
270N/A # spread out into chunks, and may require a delta-oriented update
270N/A # interface.
270N/A
270N/A def __init__(self, cat_root, pkg_root = None):
59N/A """Create a catalog. If the path supplied does not exist,
37N/A this will create the required directory structure.
59N/A Otherwise, if the directories are already in place, the
59N/A existing catalog is opened"""
59N/A
59N/A self.catalog_root = cat_root
59N/A self.attrs = {}
59N/A
157N/A self.attrs["npkgs"] = 0
157N/A
59N/A if not os.path.exists(cat_root):
59N/A os.makedirs(cat_root)
59N/A
59N/A catpath = os.path.normpath(os.path.join(cat_root, "catalog"))
59N/A
23N/A if pkg_root is not None and not os.path.exists(catpath):
270N/A self.build_catalog(pkg_root)
270N/A
157N/A self.set_time()
157N/A
157N/A def add_fmri(self, fmri, critical = False):
195N/A """Add a package, named by the fmri, to the catalog.
195N/A Throws an exception if an identical package is already
195N/A present. Throws an exception if package has no version."""
195N/A if fmri.version == None:
157N/A raise CatalogException, \
157N/A "Unversioned FMRI not supported: %s" % fmri
157N/A
195N/A if critical:
258N/A pkgstr = "C %s\n" % fmri.get_fmri(anarchy = True)
270N/A else:
204N/A pkgstr = "V %s\n" % fmri.get_fmri(anarchy = True)
207N/A
204N/A pathstr = os.path.normpath(os.path.join(self.catalog_root,
204N/A "catalog"))
204N/A
204N/A pfile = file(pathstr, "a+")
204N/A pfile.seek(0)
204N/A
204N/A for entry in pfile:
270N/A if entry == pkgstr:
204N/A pfile.close()
204N/A raise CatalogException, \
204N/A "Package is already in the catalog"
34N/A
157N/A pfile.write(pkgstr)
157N/A pfile.close()
157N/A
221N/A self.attrs["npkgs"] += 1
157N/A
270N/A self.set_time()
204N/A self.save_attrs()
204N/A
157N/A def attrs_as_lines(self):
195N/A """Takes the list of in-memory attributes and returns
220N/A a list of strings, each string naming an attribute."""
50N/A
270N/A ret = []
270N/A
270N/A for k,v in self.attrs.items():
270N/A s = "S %s: %s\n" % (k, v)
270N/A ret.append(s)
270N/A
270N/A return ret
270N/A
270N/A def build_catalog(self, pkg_root):
270N/A """Walk the on-disk package data and build (or rebuild)
270N/A the package catalog."""
270N/A tree = os.walk(pkg_root)
270N/A
270N/A # XXX eschew os.walk in favor of another os.listdir here?
270N/A for pkg in tree:
270N/A if pkg[0] == pkg_root:
270N/A continue
270N/A
270N/A for e in os.listdir(pkg[0]):
270N/A e = urllib.unquote(e)
270N/A v = version.Version(e, None)
270N/A f = fmri.PkgFmri(urllib.unquote(
270N/A os.path.basename(pkg[0])), None)
157N/A f.version = v
157N/A
157N/A self.add_fmri(f)
157N/A
157N/A print f
157N/A
157N/A def get_matching_fmris(self, patterns, matcher = None,
26N/A constraint = None, counthash = None):
258N/A """Iterate through the catalog, looking for packages matching
258N/A 'pattern', based on the function in 'matcher' and the versioning
258N/A constraint described by 'constraint'. If 'matcher' is None,
258N/A uses fmri subset matching as the default. Returns a sorted list
258N/A of PkgFmri objects, newest versions first. If 'counthash' is a
258N/A dictionary, instead store the number of matched fmris for each
258N/A package name which was matched."""
258N/A
50N/A if not matcher:
157N/A matcher = fmri.fmri_match
157N/A
157N/A if not isinstance(patterns, list):
34N/A patterns = [ patterns ]
157N/A
157N/A # 'pattern' may be a partially or fully decorated fmri; we want
157N/A # to extract its name and version to match separately against
157N/A # the catalog.
157N/A # XXX "5.11" here needs to be saner
36N/A tuples = {}
157N/A
157N/A for pattern in patterns:
157N/A if isinstance(pattern, fmri.PkgFmri):
157N/A tuples[pattern] = pattern.tuple()
207N/A else:
207N/A tuples[pattern] = \
157N/A fmri.PkgFmri(pattern, "5.11").tuple()
157N/A
157N/A ret = []
30N/A
157N/A pfile = file(os.path.normpath(
50N/A os.path.join(self.catalog_root, "catalog")), "r")
215N/A
215N/A for entry in pfile:
215N/A try:
270N/A cv, pkg, cat_name, cat_version = entry.split()
270N/A except ValueError:
270N/A # Handle old two-column catalog file, mostly in
270N/A # use on server.
215N/A cv, cat_fmri = entry.split()
157N/A pkg = "pkg"
220N/A cat_auth, cat_name, cat_version = \
220N/A fmri.PkgFmri(cat_fmri, "5.11").tuple()
220N/A
220N/A for pattern in patterns:
220N/A pat_auth, pat_name, pat_version = tuples[pattern]
220N/A if pkg == "pkg" and matcher(cat_name, pat_name):
220N/A pkgfmri = fmri.PkgFmri("%s@%s" %
220N/A (cat_name, cat_version))
220N/A if not pat_version or \
220N/A not pat_version.is_successor(
157N/A pkgfmri.version, constraint):
157N/A if counthash is not None:
157N/A if pattern in counthash:
50N/A counthash[pattern] += 1
50N/A else:
50N/A counthash[pattern] = 1
157N/A else:
157N/A ret.append(pkgfmri)
157N/A
157N/A pfile.close()
157N/A
157N/A if counthash:
204N/A return counthash
204N/A
204N/A if not ret:
204N/A raise KeyError, \
204N/A "patterns '%s' match no packages in catalog" % \
204N/A patterns
204N/A
204N/A return sorted(ret, reverse = True)
204N/A
204N/A def fmris(self):
220N/A """A generator function that produces FMRIs as it
220N/A iterates over the contents of the catalog."""
220N/A
220N/A pfile = file(os.path.normpath(
220N/A os.path.join(self.catalog_root, "catalog")), "r")
220N/A
220N/A for entry in pfile:
220N/A try:
220N/A cv, pkg, cat_name, cat_version = entry.split()
220N/A if pkg == "pkg":
220N/A yield fmri.PkgFmri("%s@%s" %
220N/A (cat_name, cat_version))
220N/A except ValueError:
220N/A # Handle old two-column catalog file, mostly in
220N/A # use on server.
220N/A cv, cat_fmri = entry.split()
220N/A yield fmri.PkgFmri(cat_fmri)
220N/A
220N/A pfile.close()
220N/A
220N/A def load_attrs(self, filenm = "attrs"):
220N/A """Load attributes from the catalog file into the in-memory
220N/A attributes dictionary"""
220N/A
220N/A apath = os.path.normpath(
220N/A os.path.join(self.catalog_root, filenm))
220N/A if not os.path.exists(apath):
220N/A return
220N/A
220N/A afile = file(apath, "r")
220N/A attrre = re.compile('^S ([^:]*): (.*)')
220N/A
220N/A for entry in afile:
220N/A m = attrre.match(entry)
220N/A if m != None:
220N/A self.attrs[m.group(1)] = m.group(2)
220N/A
220N/A afile.close()
220N/A
220N/A def npkgs(self):
204N/A """Returns the number of packages in the catalog."""
204N/A
204N/A return self.attrs["npkgs"]
204N/A
204N/A @staticmethod
204N/A def recv(filep, path):
204N/A """A class method that takes a file-like object and
204N/A a path. This is the other half of catalog.send(). It
204N/A reads a stream as an incoming catalog and lays it down
204N/A on disk."""
204N/A
204N/A if not os.path.exists(path):
204N/A os.makedirs(path)
204N/A
204N/A attrf = file(os.path.normpath(
204N/A os.path.join(path, "attrs")), "w+")
204N/A catf = file(os.path.normpath(
204N/A os.path.join(path, "catalog")), "w+")
204N/A
204N/A for s in filep:
157N/A if s.startswith("S "):
157N/A attrf.write(s)
204N/A else:
157N/A # XXX Need to be able to handle old and new
204N/A # format catalogs.
157N/A f = fmri.PkgFmri(s[2:])
157N/A catf.write("%s %s %s %s\n" %
157N/A (s[0], "pkg", f.pkg_name, f.version))
204N/A
204N/A attrf.close()
204N/A catf.close()
204N/A
204N/A def save_attrs(self, filenm = "attrs"):
204N/A """Save attributes from the in-memory catalog to a file
204N/A specified by filenm."""
204N/A
204N/A afile = file(os.path.normpath(
204N/A os.path.join(self.catalog_root, filenm)), "w+")
204N/A for a in self.attrs.keys():
204N/A s = "S %s: %s\n" % (a, self.attrs[a])
204N/A afile.write(s)
204N/A
204N/A afile.close()
204N/A
204N/A def send(self, filep):
204N/A """Send the contents of this catalog out to the filep
204N/A specified as an argument."""
204N/A
204N/A # Send attributes first.
204N/A filep.writelines(self.attrs_as_lines())
204N/A
264N/A try:
264N/A cfile = file(os.path.normpath(
264N/A os.path.join(self.catalog_root, "catalog")), "r")
264N/A except IOError, e:
264N/A # Missing catalog is fine; other errors need to be
264N/A # reported.
264N/A if e.errno == errno.ENOENT:
264N/A return
264N/A else:
264N/A raise
264N/A
264N/A for e in cfile:
264N/A filep.write(e)
264N/A
264N/A cfile.close()
264N/A
264N/A def set_time(self):
264N/A self.attrs["Last-Modified"] = time.strftime("%Y%m%dT%H%M%SZ")
264N/A
264N/A
204N/A# In order to avoid a fine from the Department of Redundancy Department,
204N/A# allow these methods to be invoked without explictly naming the Catalog class.
204N/Arecv = Catalog.recv
204N/A
204N/Aif __name__ == "__main__":
204N/A cpath = tempfile.mkdtemp()
204N/A c = Catalog(cpath)
204N/A
204N/A for f in [
204N/A fmri.PkgFmri("pkg:/test@1.0,5.11-1:20000101T120000Z", None),
204N/A fmri.PkgFmri("pkg:/test@1.0,5.11-1:20000101T120010Z", None),
204N/A fmri.PkgFmri("pkg:/test@1.0,5.11-1.1:20000101T120020Z", None),
204N/A fmri.PkgFmri("pkg:/test@1.0,5.11-1.2:20000101T120030Z", None),
204N/A fmri.PkgFmri("pkg:/test@1.0,5.11-2:20000101T120040Z", None),
204N/A fmri.PkgFmri("pkg:/test@1.1,5.11-1:20000101T120040Z", None),
264N/A fmri.PkgFmri("pkg:/apkg@1.0,5.11-1:20000101T120040Z", None),
264N/A fmri.PkgFmri("pkg:/zpkg@1.0,5.11-1:20000101T120040Z", None)
264N/A ]:
264N/A c.add_fmri(f)
264N/A
274N/A print c
264N/A
264N/A tps = [
264N/A "pkg:/test@1.0,5.10-1:20070101T120000Z",
264N/A "pkg:/test@1.0,5.11-1:20061231T120000Z",
264N/A "pkg:/test@1.0,5.11-2",
264N/A "pkg:/test@1.0,5.11-3"
264N/A ]
264N/A
264N/A for tp in tps:
264N/A print "matches for %s:" % tp
264N/A
264N/A for p in c.get_matching_fmris(tp, None):
264N/A print " ", p
264N/A
264N/A # Print TPS report
264N/A for p in c.get_matching_fmris("test", matcher = fmri.regex_match):
264N/A print p
264N/A
264N/A try:
204N/A l = c.get_matching_fmris("flob", matcher = fmri.regex_match)
204N/A except KeyError:
204N/A print "correctly determined no match for 'flob'"
204N/A
204N/A for f in c.fmris():
204N/A print f
204N/A
204N/A shutil.rmtree(cpath)
265N/A