catalog.py revision 195
23N/A#!/usr/bin/python
23N/A#
23N/A# CDDL HEADER START
23N/A#
23N/A# The contents of this file are subject to the terms of the
23N/A# Common Development and Distribution License (the "License").
23N/A# You may not use this file except in compliance with the License.
23N/A#
23N/A# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
23N/A# or http://www.opensolaris.org/os/licensing.
23N/A# See the License for the specific language governing permissions
23N/A# and limitations under the License.
23N/A#
23N/A# When distributing Covered Code, include this CDDL HEADER in each
23N/A# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
23N/A# If applicable, add the following below this CDDL HEADER, with the
23N/A# fields enclosed by brackets "[]" replaced with your own identifying
23N/A# information: Portions Copyright [yyyy] [name of copyright owner]
23N/A#
23N/A# CDDL HEADER END
23N/A#
221N/A# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
23N/A# Use is subject to license terms.
23N/A
221N/Aimport os
221N/Aimport re
221N/Aimport sha
23N/Aimport shutil
23N/Aimport time
23N/Aimport urllib
174N/Aimport tempfile
204N/Aimport errno
204N/A
204N/Aimport pkg.fmri as fmri
215N/Aimport pkg.version as version
264N/A
270N/Aclass CatalogException(Exception):
23N/A def __init__(self, args=None):
23N/A self.args = args
157N/A
204N/Aclass Catalog(object):
204N/A """A Catalog is the representation of the package FMRIs available to
157N/A this client or repository. Both purposes utilize the same storage
157N/A format.
157N/A
157N/A The serialized structure of the repository is an unordered list of
23N/A available package versions, followed by an unordered list of
258N/A incorporation relationships between packages. This latter section
258N/A allows the graph to be topologically sorted by the client.
258N/A
258N/A S Last-Modified: [timespec]
34N/A
34N/A XXX A authority mirror-uri ...
34N/A XXX ...
34N/A
24N/A V fmri
26N/A V fmri
26N/A ...
26N/A C fmri
26N/A C fmri
26N/A ...
50N/A I fmri fmri
50N/A I fmri fmri
37N/A ...
37N/A """
37N/A
26N/A # XXX Mirroring records also need to be allowed from client
26N/A # configuration, and not just catalogs.
26N/A #
50N/A # XXX It would be nice to include available tags and package sizes,
50N/A # although this could also be calculated from the set of manifests.
50N/A #
26N/A # XXX Current code is O(N_packages) O(M_versions), should be
26N/A # O(1) O(M_versions), and possibly O(1) O(1).
26N/A #
270N/A # XXX Initial estimates suggest that the Catalog could be composed of
270N/A # 1e5 - 1e7 lines. Catalogs across these magnitudes will need to be
270N/A # spread out into chunks, and may require a delta-oriented update
270N/A # interface.
270N/A
270N/A def __init__(self, cat_root, authority = None, pkg_root = None):
59N/A """Create a catalog. If the path supplied does not exist,
37N/A this will create the required directory structure.
59N/A Otherwise, if the directories are already in place, the
59N/A existing catalog is opened. If pkg_root is specified
59N/A and no catalog is found at cat_root, the catalog will be
59N/A rebuilt. authority names the authority that
59N/A is represented by this catalog."""
59N/A
157N/A self.catalog_root = cat_root
157N/A self.attrs = {}
59N/A self.auth = authority
59N/A
59N/A self.attrs["npkgs"] = 0
59N/A
59N/A if not os.path.exists(cat_root):
23N/A os.makedirs(cat_root)
270N/A
270N/A catpath = os.path.normpath(os.path.join(cat_root, "catalog"))
157N/A
157N/A if pkg_root is not None and not os.path.exists(catpath):
157N/A self.build_catalog(pkg_root)
195N/A self.set_time()
195N/A self.save_attrs()
195N/A
195N/A self.load_attrs()
157N/A
157N/A def add_fmri(self, fmri, critical = False):
157N/A """Add a package, named by the fmri, to the catalog.
195N/A Throws an exception if an identical package is already
258N/A present. Throws an exception if package has no version."""
270N/A if fmri.version == None:
204N/A raise CatalogException, \
207N/A "Unversioned FMRI not supported: %s" % fmri
204N/A
204N/A if critical:
204N/A pkgstr = "C %s\n" % fmri.get_fmri(anarchy = True)
204N/A else:
204N/A pkgstr = "V %s\n" % fmri.get_fmri(anarchy = True)
204N/A
204N/A pathstr = os.path.normpath(os.path.join(self.catalog_root,
270N/A "catalog"))
204N/A
204N/A pfile = file(pathstr, "a+")
204N/A pfile.seek(0)
34N/A
157N/A for entry in pfile:
157N/A if entry == pkgstr:
157N/A pfile.close()
221N/A raise CatalogException, \
157N/A "Package is already in the catalog"
270N/A
204N/A pfile.write(pkgstr)
204N/A pfile.close()
157N/A
195N/A self.attrs["npkgs"] += 1
220N/A
50N/A self.set_time()
270N/A self.save_attrs()
270N/A
270N/A def attrs_as_lines(self):
270N/A """Takes the list of in-memory attributes and returns
270N/A a list of strings, each string naming an attribute."""
270N/A
270N/A ret = []
270N/A
270N/A for k,v in self.attrs.items():
270N/A s = "S %s: %s\n" % (k, v)
270N/A ret.append(s)
270N/A
270N/A return ret
270N/A
270N/A def build_catalog(self, pkg_root):
270N/A """Walk the on-disk package data and build (or rebuild)
270N/A the package catalog."""
270N/A tree = os.walk(pkg_root)
270N/A
270N/A # XXX eschew os.walk in favor of another os.listdir here?
270N/A for pkg in tree:
270N/A if pkg[0] == pkg_root:
270N/A continue
157N/A
157N/A for e in os.listdir(pkg[0]):
157N/A e = urllib.unquote(e)
157N/A v = version.Version(e, None)
157N/A f = fmri.PkgFmri(urllib.unquote(
157N/A os.path.basename(pkg[0])), None)
157N/A f.version = v
26N/A
258N/A self.add_fmri(f)
258N/A
258N/A print f
258N/A
258N/A def get_matching_fmris(self, patterns, matcher = None,
258N/A constraint = None, counthash = None):
258N/A """Iterate through the catalog, looking for packages matching
258N/A 'pattern', based on the function in 'matcher' and the versioning
50N/A constraint described by 'constraint'. If 'matcher' is None,
157N/A uses fmri subset matching as the default. Returns a sorted list
157N/A of PkgFmri objects, newest versions first. If 'counthash' is a
157N/A dictionary, instead store the number of matched fmris for each
34N/A package name which was matched."""
157N/A
157N/A cat_auth = self.auth
157N/A
157N/A if not matcher:
157N/A matcher = fmri.fmri_match
36N/A
157N/A if not isinstance(patterns, list):
157N/A patterns = [ patterns ]
157N/A
157N/A # 'pattern' may be a partially or fully decorated fmri; we want
207N/A # to extract its name and version to match separately against
207N/A # the catalog.
157N/A # XXX "5.11" here needs to be saner
157N/A tuples = {}
157N/A
30N/A for pattern in patterns:
157N/A if isinstance(pattern, fmri.PkgFmri):
50N/A tuples[pattern] = pattern.tuple()
215N/A else:
215N/A tuples[pattern] = \
215N/A fmri.PkgFmri(pattern, "5.11").tuple()
270N/A
270N/A ret = []
270N/A
270N/A try:
215N/A pfile = file(os.path.normpath(
157N/A os.path.join(self.catalog_root, "catalog")), "r")
220N/A except IOError, e:
220N/A if e.errno == errno.ENOENT:
220N/A return ret
220N/A else:
220N/A raise
220N/A
220N/A for entry in pfile:
220N/A try:
220N/A cv, pkg, cat_name, cat_version = entry.split()
220N/A except ValueError:
157N/A # Handle old two-column catalog file, mostly in
157N/A # use on server.
157N/A cv, cat_fmri = entry.split()
50N/A pkg = "pkg"
50N/A cat_auth, cat_name, cat_version = \
50N/A fmri.PkgFmri(cat_fmri, "5.11",
157N/A authority = self.auth).tuple()
157N/A
157N/A for pattern in patterns:
157N/A pat_auth, pat_name, pat_version = tuples[pattern]
157N/A if pkg == "pkg" and \
157N/A (pat_auth == cat_auth or not pat_auth) and \
204N/A matcher(cat_name, pat_name):
204N/A pkgfmri = fmri.PkgFmri("%s@%s" %
204N/A (cat_name, cat_version),
204N/A authority = cat_auth)
204N/A if not pat_version or \
204N/A pkgfmri.version.is_successor(
204N/A pat_version, constraint) or \
204N/A pkgfmri.version == pat_version:
204N/A if counthash is not None:
204N/A if pattern in counthash:
220N/A counthash[pattern] += 1
220N/A else:
220N/A counthash[pattern] = 1
220N/A ret.append(pkgfmri)
220N/A
220N/A pfile.close()
220N/A
220N/A return sorted(ret, reverse = True)
220N/A
220N/A def fmris(self):
220N/A """A generator function that produces FMRIs as it
220N/A iterates over the contents of the catalog."""
220N/A
220N/A try:
220N/A pfile = file(os.path.normpath(
220N/A os.path.join(self.catalog_root, "catalog")), "r")
220N/A except IOError, e:
220N/A if e.errno == errno.ENOENT:
220N/A return
220N/A else:
220N/A raise
220N/A
220N/A for entry in pfile:
220N/A try:
220N/A cv, pkg, cat_name, cat_version = entry.split()
220N/A if pkg == "pkg":
220N/A yield fmri.PkgFmri("%s@%s" %
220N/A (cat_name, cat_version),
220N/A authority = self.auth)
220N/A except ValueError:
220N/A # Handle old two-column catalog file, mostly in
220N/A # use on server.
220N/A cv, cat_fmri = entry.split()
220N/A yield fmri.PkgFmri(cat_fmri,
220N/A authority = self.auth)
220N/A
220N/A pfile.close()
220N/A
220N/A def load_attrs(self, filenm = "attrs"):
220N/A """Load attributes from the catalog file into the in-memory
204N/A attributes dictionary"""
204N/A
204N/A apath = os.path.normpath(
204N/A os.path.join(self.catalog_root, filenm))
204N/A if not os.path.exists(apath):
204N/A return
204N/A
204N/A afile = file(apath, "r")
204N/A attrre = re.compile('^S ([^:]*): (.*)')
204N/A
204N/A for entry in afile:
204N/A m = attrre.match(entry)
204N/A if m != None:
204N/A self.attrs[m.group(1)] = m.group(2)
204N/A
204N/A afile.close()
204N/A
204N/A if "npkgs" in self.attrs:
204N/A self.attrs["npkgs"] = int(self.attrs["npkgs"])
204N/A
157N/A def npkgs(self):
157N/A """Returns the number of packages in the catalog."""
204N/A
157N/A return self.attrs["npkgs"]
204N/A
157N/A @staticmethod
157N/A def recv(filep, path):
157N/A """A class method that takes a file-like object and
204N/A a path. This is the other half of catalog.send(). It
204N/A reads a stream as an incoming catalog and lays it down
204N/A on disk."""
204N/A
204N/A if not os.path.exists(path):
204N/A os.makedirs(path)
204N/A
204N/A attrf = file(os.path.normpath(
204N/A os.path.join(path, "attrs")), "w+")
204N/A catf = file(os.path.normpath(
204N/A os.path.join(path, "catalog")), "w+")
204N/A
204N/A for s in filep:
204N/A if s.startswith("S "):
204N/A attrf.write(s)
204N/A else:
204N/A # XXX Need to be able to handle old and new
204N/A # format catalogs.
204N/A f = fmri.PkgFmri(s[2:])
204N/A catf.write("%s %s %s %s\n" %
204N/A (s[0], "pkg", f.pkg_name, f.version))
204N/A
204N/A attrf.close()
264N/A catf.close()
264N/A
264N/A def save_attrs(self, filenm = "attrs"):
264N/A """Save attributes from the in-memory catalog to a file
264N/A specified by filenm."""
264N/A
264N/A afile = file(os.path.normpath(
264N/A os.path.join(self.catalog_root, filenm)), "w+")
264N/A for a in self.attrs.keys():
264N/A s = "S %s: %s\n" % (a, self.attrs[a])
264N/A afile.write(s)
264N/A
264N/A afile.close()
264N/A
264N/A def send(self, filep):
264N/A """Send the contents of this catalog out to the filep
264N/A specified as an argument."""
264N/A
264N/A # Send attributes first.
264N/A filep.writelines(self.attrs_as_lines())
204N/A
204N/A try:
204N/A cfile = file(os.path.normpath(
204N/A os.path.join(self.catalog_root, "catalog")), "r")
204N/A except IOError, e:
204N/A # Missing catalog is fine; other errors need to be
204N/A # reported.
204N/A if e.errno == errno.ENOENT:
204N/A return
204N/A else:
204N/A raise
204N/A
204N/A for e in cfile:
204N/A filep.write(e)
204N/A
264N/A cfile.close()
264N/A
264N/A def set_time(self):
264N/A self.attrs["Last-Modified"] = time.strftime("%Y%m%dT%H%M%SZ")
264N/A
274N/A
264N/A# In order to avoid a fine from the Department of Redundancy Department,
264N/A# allow these methods to be invoked without explictly naming the Catalog class.
264N/Arecv = Catalog.recv
264N/A