#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
#
import errno
import os
import platform
import shutil
# Constants for indicating whether pkgplans or fmri-manifest path pairs are
# used as arguments.
"""Create a directory at the specified location if it does not
already exist (including any parent directories).
"""
try:
except EnvironmentError as e:
return
e.filename)
raise
"""Indexer is a class designed to index a set of manifests or pkg plans
and provide a compact representation on disk, which is quickly
searchable."""
raise search_errors.IndexingException(
_("sort_file_max_size must be greater than 0"))
# This structure was used to gather all index files into one
# location. If a new index structure is needed, the files can
# be added (or removed) from here. Providing a list or
# dictionary allows an easy approach to opening or closing all
# index files.
self._data_dict = {
"fast_add":
"fast_remove":
"manf":
"token_byte_offset":
}
# This is added to the dictionary after the others because it
# needs one of the other mappings as an input.
self.file_version_number = None
if progtrack is None:
else:
# The action type and key indexes, which are necessary for
# efficient searches by type or key, store their file handles in
# dictionaries. File handles for actions are in at_fh, while
# filehandles for keys are kept in st_fh.
self.old_out_token = None
"""Turn fmris into strings correctly while writing out
the fmri offsets file."""
def __build_fmri(s):
"""Build fmris while reading the fmri offset information."""
""" Private method for building versions from a string. """
""" Opens all index files using consistent_open and reads all
of them into memory except the main dictionary file to avoid
inefficient memory usage."""
if res == None:
return None
try:
try:
if (d == self._data_main_dict or
d == self._data_token_offset):
continue
d.read_dict_file()
except:
raise
finally:
if d == self._data_main_dict:
continue
"""Utility fuction used to close and sort the temporary
files used to produce a sorted main_dict file."""
l = [
line)
]
l.sort()
"""Adds tokens, and the actions generating them, to the current
temporary sort file.
The "pfmri" parameter is the fmri the information is coming
from.
The "new_dict" parameter maps tokens to the information about
the action."""
lst)
return
"""Updates the log of packages which have been installed or
removed since the last time the index has been rebuilt.
There are two axes to consider: whether the package is being
added or removed; whether this version of the package is
already present in an update log.
Case 1: The package is being installed and is not in the
update log. In this case, the new package is simply added
to the install update log.
Case 2: The package is being installed and is in the removal
update log. In this case, the package is removed from the
remove update log. This has the effect of exposing the
entries in the existing index to the user.
Case 3: The package is being removed and is not in the
update log. In this case, the new package is simply added
to the removed update log.
Case 4: The package is being removed and is in the installed
update log. In this case, the package is removed from the
install update log.
The "filters_pkgplan_list" parameter is a tuple of a list of
filters, which are currently ignored, and a list of pkgplans
that indicated which versions of a package are being added or
removed."""
#
# First pass determines whether a fast update makes sense and
# updates the list of fmris that will be in the index.
#
for p in pkgplan_list:
if d_fmri:
nfast_remove -= 1
else:
nfast_add += 1
if o_fmri:
nfast_add -= 1
else:
nfast_remove += 1
return False
#
# Second pass actually updates the fast_add and fast_remove
# sets and updates progress.
#
for p in pkgplan_list:
if d_fmri:
else:
if o_fmri:
else:
return True
"""Takes a list of fmris and updates the internal storage to
reflect the new packages."""
removed_paths = []
for added_fmri in fmris:
return removed_paths
"""Writes out the new main dictionary file and also adds the
token offsets to _data_token_offset. file_handle is the file
handle for the output main dictionary file. token is the token
to add to the file. fv_fmri_pos_list_list is a structure of
lists inside of lists several layers deep. The top layer is a
list of action types. The second layer contains the keys for
the action type it's a sublist for. The third layer contains
the values which matched the token for the action and key it's
contained in. The fourth layer is the fmris which contain those
matches. The fifth layer is the offset into the manifest of
each fmri for each matching value. out_dir points to the
base directory to use to write a file for each package which
contains the offsets into the main dictionary for the tokens
this package matches."""
if self.old_out_token is not None and \
raise RuntimeError("In writing dict line, token:{0}, "
"""Takes two arguments. Each of the arguments must be a list
with the type signature list of ('a, list of 'b). Where
the lists share a value (A) for 'a, it splices the lists of 'b
paired with A from each list into a single list and makes that
the new list paired with A in the result.
Note: This modifies the ret_list rather than building a new one
because of the large performance difference between the two
approaches."""
tmp_res = []
break
if not found:
"""Produces a stream of ordered tokens and the associated
information for those tokens from the sorted temporary files
produced by _add_terms. In short, this is the merge part of the
merge sort being done on the tokens to be indexed."""
"""Helper function to make the initialization of the
fh_dict easier to understand."""
try:
return \
except StopIteration:
return None
# Build a mapping from numbers to the file handle for the
# temporary sort file with that number.
])
cur_toks = {}
# Seed cur_toks with the first token from each temporary file.
# The line may not exist since, for a empty repo, an empty file
# is created.
if line is None:
del fh_dict[i]
else:
old_min_token = None
# cur_toks will have items deleted from it as files no longer
# have tokens to provide. When no files have tokens, the merge
# is done.
while cur_toks:
min_token = None
matches = []
# Find smallest available token and the temporary files
# which contain that token.
if cur_tok is None:
continue
matches = [i]
assert min_token is not None
res = None
for i in matches:
try:
# Continue pulling the next tokens from
# and adding them to the result list as
# long as the token matches min_token.
if res is None:
else:
except StopIteration:
# When a StopIteration happens, the
# last line in the file has been read
# and processed. Delete all the
# information associated with that file
# so that we no longer check that file.
del fh_dict[i]
del cur_toks[i]
assert res is not None
if old_min_token is not None and \
raise RuntimeError("Got min token:{0} greater "
"than old_min_token:{1}".format(
if min_token != "":
return
"""Processes the main dictionary file and writes out a new
main dictionary file reflecting the changes in the packages.
The "dicts" parameter is the list of fmris which have been
removed during update.
The "out_dir" parameter is the temporary directory in which to
build the indexes."""
if self.empty_index:
file_handle = []
else:
assert file_handle
if self.file_version_number == None:
else:
# The dictionary file's opened in append mode to avoid removing
# the version information the search storage class added.
try:
except StopIteration:
try:
for line in file_handle:
line)
existing_entries = []
st_res = []
fv_res = []
p_res = []
if pfmri not in removed_paths:
if p_res:
if fv_res:
if st_res:
# Add tokens newly discovered in the added
# packages which are alphabetically earlier
# than the token most recently read from the
# existing main dictionary file.
try:
next_new_tok, new_tok_info = \
except StopIteration:
del next_new_tok
del new_tok_info
# Combine the information about the current
# token from the new packages with the existing
# information for that token.
try:
next_new_tok, new_tok_info = \
except StopIteration:
del next_new_tok
del new_tok_info
# If this token has any packages still
# associated with it, write them to the file.
if existing_entries:
# For any new tokens which are alphabetically after the
# last entry in the existing file, add them to the end
# of the file.
while new_toks_available:
try:
next_new_tok, new_tok_info = \
except StopIteration:
finally:
if not self.empty_index:
removed_paths = []
"""Write out the companion dictionaries needed for
translating the internal representation of the main
dictionary into human readable information.
The "out_dir" parameter is the temporary directory to write
the indexes into."""
if d == self._data_main_dict or \
d == self._data_token_offset:
continue
tmp_index_dir=None, image=None):
"""Performs all the steps needed to update the indexes.
The "inputs" parameter iterates over the fmris which have been
added or the pkgplans for the change in the image.
The "input_type" paramter is a value specifying whether the
input is fmris or pkgplans.
The "tmp_index_dir" parameter allows this function to use a
different temporary directory than the default.
The "image" parameter must be set if "input_type" is pkgplans.
It allows the index to automatically be rebuilt if the number
of packages added since last index rebuild is greater than
MAX_ADDED_NUMBER_PACKAGES."""
try:
# Allow the use of a directory other than the default
# directory to store the intermediate results in.
if not tmp_index_dir:
# Read the existing dictionaries.
except:
raise
try:
# If the temporary indexing directory already exists,
# remove it to ensure its empty. Since the caller
# should have locked the index already, this should
# be safe.
# Create directory.
if input_type == IDX_INPUT_TYPE_PKG:
assert image
#
# Try to do a fast update; if that fails,
# do a full index rebuild.
#
if not fast_update:
# Before passing control to rebuild
# index, the index lock must be
# released.
return self.rebuild_index_from_scratch(
elif input_type == IDX_INPUT_TYPE_FMRI:
# Update the main dictionary file
else:
raise RuntimeError(
"Got unknown input_type: {0}", input_type)
# Write out the helper dictionaries
# Move all files from the tmp directory into the index
# dir. Note: the need for consistent_open is that
# migrate is not an atomic action.
except:
raise
finally:
"""This version of update index is designed to work with the
client side of things. Specifically, it expects a pkg plan
tmp_index_dir is specified, it must NOT exist in the current
directory structure. This prevents the indexer from
accidentally removing files. Image the image object. This is
needed to allow correct reindexing from scratch to occur."""
""" This version of update index is designed to work with the
server side of things. Specifically, since we don't currently
support removal of a package from a repo, this function simply
takes a list of FMRIs to be added to the repot. Currently, the
only way to remove a package from the index is to remove it
from the depot and reindex. Note: if tmp_index_dir is
specified, it must NOT exist in the current directory structure.
This prevents the indexer from accidentally removing files."""
""" Returns a boolean value indicating whether a consistent
index exists. If an index exists but is inconsistent, an
exception is raised."""
try:
try:
res = \
except (KeyboardInterrupt,
raise
except Exception:
return False
finally:
assert res is not 0
return res
"""Removes any existing index directory and rebuilds the
index based on the fmris and manifests provided as an
argument.
The "tmp_index_dir" parameter allows for a different directory
than the default to be used."""
# A lock can't be held while the index directory is being
# removed as that can cause rmtree() to fail when using
# NFS. As such, attempt to get the lock first, then
# unlock, immediately rename the old index directory,
# and then remove the old the index directory and
# create a new one.
try:
except OSError as e:
"""Seeds the index directory with empty stubs if the directory
is consistently empty. Does not overwrite existing indexes."""
d.get_file_name())
else:
if present:
return
raise RuntimeError("Got file_version_number other than "
"None in setup.")
"""Check to see whether the catalog has fmris which have not
been indexed.
'index_root' is the path to the index to check against.
'cat' is the catalog to check for new fmris."""
try:
except IOError as e:
return fmri_set
else:
raise
try:
finally:
return fmri_set
"""Moves the indexes from a temporary directory to the
permanent one.
The "source_dir" parameter is the directory containing the
new information.
The "dest_dir" parameter is the directory containing the
old information.
The "fast_update" parameter determines whether the main
dictionary and the token byte offset files are moved. This is
used so that when only the update logs are touched, the large
files don't need to be moved."""
if not source_dir:
if not dest_dir:
assert not (source_dir == dest_dir)
d == self._data_token_offset or
d == self._data_fmri_offsets):
continue
else:
d.get_file_name()),
if not fast_update:
# by the fmri_offsets.v1 file.
try:
except KeyboardInterrupt:
raise
except Exception:
pass
"""Locks the index in preparation for an index-modifying
operation. Raises an IndexLockedException exception on
failure.
'blocking' is an optional boolean value indicating whether
to block until the lock can be obtained or to raise an
exception immediately if it cannot be."""
try:
# Attempt to obtain a file lock.
except EnvironmentError as e:
# If a lock was requested, and the only
# reason for failure was because the
# index directory doesn't exist yet,
# then create it and try once more.
raise
raise search_errors.\
e.filename)
raise
"""Unlocks the index."""