96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# CDDL HEADER START
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# The contents of this file are subject to the terms of the
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Common Development and Distribution License (the "License").
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# You may not use this file except in compliance with the License.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# See the License for the specific language governing permissions
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# and limitations under the License.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# When distributing Covered Code, include this CDDL HEADER in each
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# If applicable, add the following below this CDDL HEADER, with the
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# fields enclosed by brackets "[]" replaced with your own identifying
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# information: Portions Copyright [yyyy] [name of copyright owner]
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# CDDL HEADER END
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# wsdiff(1) is a tool that can be used to determine which compiled objects
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# have changed as a result of a given source change. Developers backporting
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# new features, RFEs and bug fixes need to be able to identify the set of
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# patch deliverables necessary for feature/fix realization on a patched system.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# The tool works by comparing objects in two trees/proto areas (one build with,
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# and without the source changes.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Using wsdiff(1) is fairly simple:
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# - Bringover to a fresh workspace
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# - Perform a full non-debug build (clobber if workspace isn't fresh)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# - Move the proto area aside, call it proto.old, or something.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# - Integrate your changes to the workspace
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# - Perform another full non-debug clobber build.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# - Use wsdiff(1) to see what changed:
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# $ wsdiff proto.old proto
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# By default, wsdiff will print the list of changed objects / deliverables to
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# stdout. If a results file is specified via -r, the list of differing objects,
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# and details about why wsdiff(1) thinks they are different will be logged to
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# the results file.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# By invoking nightly(1) with the -w option to NIGHTLY_FLAGS, nightly(1) will use
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# wsdiff(1) to report on what objects changed since the last build.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# For patch deliverable purposes, it's advised to have nightly do a clobber,
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# non-debug build.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Think about the results. Was something flagged that you don't expect? Go look
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# at the results file to see details about the differences.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Use the -i option in conjunction with -v and -V to dive deeper and have wsdiff(1)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# report with more verbosity.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Usage: wsdiff [-vVt] [-r results ] [-i filelist ] old new
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Where "old" is the path to the proto area build without the changes, and
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# "new" is the path to the proto area built with the changes. The following
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# options are supported:
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -v Do not truncate observed diffs in results
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -V Log *all* ELF sect diffs vs. logging the first diff found
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -t Use onbld tools in $SRC/tools
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -r Log results and observed differences
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -i Tell wsdiff which objects to compare via an input file list
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotalimport datetime, fnmatch, getopt, os, profile, commands
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotalimport re, resource, select, shutil, signal, string, struct, sys, tempfile
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxefrom stat import *
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Human readable diffs truncated by default if longer than this
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Specifying -v on the command line will override
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# Lock name Provides exclusive access to
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# --------------+------------------------------------------------
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# output_lock standard output or temporary file (difference())
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# log_lock the results file (log_difference())
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# wset_lock changedFiles list (workerThread())
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# Variable for thread control
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Default search path for wsdiff
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# These are objects that wsdiff will notice look different, but will not report.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Existence of an exceptions list, and adding things here is *dangerous*,
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# and therefore the *only* reasons why anything would be listed here is because
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# the objects do not build deterministically, yet we *cannot* fix this.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# These perl libraries use __DATE__ and therefore always look different.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Ideally, we would purge use the use of __DATE__ from the source, but because
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# this is source we wish to distribute with Solaris "unchanged", we cannot modify.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxewsdiff_exceptions = [ "usr/perl5/5.8.4/lib/sun4-solaris-64int/CORE/libperl.so.1",
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe "usr/perl5/5.6.1/lib/sun4-solaris-64int/CORE/libperl.so.1",
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe "usr/perl5/5.8.4/lib/i86pc-solaris-64int/CORE/libperl.so.1",
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe "usr/perl5/5.6.1/lib/i86pc-solaris-64int/CORE/libperl.so.1"
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Logging routines
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# Debug message to be printed to the screen, and the log file
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Add prefix to highlight debugging message
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Informational message to be printed to the screen, and the log file
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Error message to be printed to the screen, and the log file
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Informational message to be printed only to the log, if there is one.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Flag a detected file difference
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Display the fileName to stdout, and log the difference
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Do the actual logging of the difference to the results file
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe print >> log, f
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe print >> log, \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe "... truncated due to length: " \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe "use -v to override ..."
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# diff generating routines
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return human readable diffs from two temporary files
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Filter the data through od(1) if the data is detected
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # as being binary
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe cmd = od_cmd + " -c -t x4" + " " + tmpf1 + " > " + tmp_od1
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe cmd = od_cmd + " -c -t x4" + " " + tmpf2 + " > " + tmp_od2
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal data = commands.getoutput(diff_cmd + " " + tmpf1 + " " + tmpf2)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Remove the temp files as we no longer need them.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("failed to get output of command: " + diff_cmd + " " \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Send exception for the failed command up
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return human readable diffs betweeen two datasets
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal tmpFile1 = tmpDir1 + os.path.basename(base) + t.getName()
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName()
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Misc utility functions
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Prune off the leading prefix from string s
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Prune off leading proto path goo (if there is one) to yield
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# the deliverable's eventual path relative to root
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# e.g. proto.base/root_sparc/usr/src/cmd/prstat => usr/src/cmd/prstat
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Usage / argument processing
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Display usage message
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal print >> sys.stderr, """Usage: wsdiff [-dvVst] [-r results ] [-i filelist ] old new
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal -d Print debug messages about the progress
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe -v Do not truncate observed diffs in results
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe -V Log *all* ELF sect diffs vs. logging the first diff found
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe -t Use onbld tools in $SRC/tools
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe -r Log results and observed differences
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal -s Produce sorted list of differences
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe -i Tell wsdiff which objects to compare via an input file list"""
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Process command line options
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # getopt.getopt() returns:
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # a list of remaining non-option arguments
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # A correct wsdiff invocation will have exactly two non option
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # arguments, the paths to the base (old), ptch (new) proto areas
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe optlist, args = getopt.getopt(sys.argv[1:], validOpts)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # alphabetical order
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe return baseRoot, fileNamesFile, localTools, ptchRoot, results
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# File identification
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Identify the file type.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# If it's not ELF, use the file extension to identify
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# certain file types that require special handling to
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# compare. Otherwise just return a basic "ASCII" type.
619b4598e0f434792bdadf91cc947739dc734758rotondo return 'ASCII'
619b4598e0f434792bdadf91cc947739dc734758rotondo return 'Error'
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe return 'ELF'
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe return 'ASCII'
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return non-zero if "f" is an ELF file
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return non-zero is "f" is binary.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Consider the file to be binary if it contains any null characters
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Directory traversal and file finding
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return a sorted list of files found under the specified directory
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Examine all files in base, ptch
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return a list of files appearing in both proto areas,
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# a list of new files (files found only in ptch) and
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# a list of deleted files (files found only in base)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Getting the list of files in the base area");
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Found " + str(len(baseFilesList)) + " files")
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Getting the list of files in the patch area");
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Found " + str(len(ptchFilesList)) + " files")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Inventory files in the base proto area
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Determining the list of regular files in the base area");
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Found " + str(len(compFiles)) + " files")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Inventory files in the patch proto area
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Determining the list of regular files in the patch area");
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Found " + str(len(ptchList)) + " files")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Deleted files appear in the base area, but not the patch area
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Searching for deleted files by comparing the lists")
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Found " + str(len(deletedFiles)) + " deleted files")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Eliminate "deleted" files from the list of objects appearing
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # in both the base and patch proto areas
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Eliminating deleted files from the list of objects")
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("List for comparison reduced to " + str(len(compFiles)) \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # New files appear in the patch area, but not the base
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Getting the list of newly added files")
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Found " + str(len(newFiles)) + " new files")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Examine the files listed in the input file list
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return a list of files appearing in both proto areas,
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# a list of new files (files found only in ptch) and
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# a list of deleted files (files found only in base)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe if f == '\n' :
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # the fileNames have a trailing '\n'
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # The objects in the file list have paths relative
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # to $ROOT or to the base/ptch directory specified on
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # the command line.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # If it's relative to $ROOT, we'll need to add back the
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # root_`uname -p` goo we stripped off in fnFormat()
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe elif os.path.exists(base + "root_" + arch + "/" + f) :
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe elif os.path.exists(ptch + "root_" + arch + "/" + f) :
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error(f + " in file list, but not in either tree. " + \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal "Skipping...")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Build a fully qualified path to an external tool/utility.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Consider the default system locations. For onbld tools, if
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# the -t option was specified, we'll try to use built tools in $SRC tools,
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# and otherwise, we'll fall back on /opt/onbld/
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # First, check what was passed
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Next try in wsdiff path
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# ELF file comparison helper routines
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return a dictionary of ELF section types keyed by section name
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe hstring = commands.getoutput(elfdump_cmd + " -c " + f)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # elfdump(1) dumps the section headers with the section name
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # following "sh_name:", and the section type following "sh_type:"
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe " in " + f)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Extract data in the specified ELF section from the given file
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe data = commands.getoutput(dump_cmd + " -sn " + section + " " + f)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error(dump_cmd + "yielded no data on section " + section + \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # dump(1) displays the file name to start...
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # get past it to the data itself
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Return a (hopefully meaningful) human readable set of diffs
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# for the specified ELF section between f1 and f2
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Depending on the section, various means for dumping and diffing
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# the data may be employed.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal tmpFile1 = tmpDir1 + os.path.basename(f1) + t.getName()
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal tmpFile2 = tmpDir2 + os.path.basename(f2) + t.getName()
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal cmd1 = elfdump_cmd + " -s -N " + section + " " + f1 + \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal cmd2 = elfdump_cmd + " -s -N " + section + " " + f2 + \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # dis sometimes complains when it hits something it doesn't
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # know how to disassemble. Just ignore it, as the output
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # being generated here is human readable, and we've already
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # correctly flagged the difference.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # remove temp files as we no longer need them
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("diff_elf_section: unlink failed %s" % e)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("diff_elf_section: unlink failed %s" % e)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# compare the relevant sections of two ELF binaries
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# and report any differences
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Returns: 1 if any differenes found
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# 0 if no differences found
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -1 on error
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Sections deliberately not considered when comparing two ELF
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# binaries. Differences observed in these sections are not considered
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# significant where patch deliverable identification is concerned.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe ".comment",
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe ".SUNW_ctf",
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe ".rela.bss",
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe ".rela.plt",
f6a1d7963180c91d5a022b2e7df19e2772fee46aesaxe ".compcom",
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe ".rodata.str1.1",
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Derive the list of ELF sections found only in
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # either e1 or e2.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Look for preferred sections, and put those at the
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # top of the list of sections to compare
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Compare ELF sections
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # If all ELF sections are being reported, then
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # invoke difference() to flag the file name to
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # stdout only once. Any other section differences
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # should be logged to the results file directly
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# recursively remove 2 directories
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# Used for removal of temporary directory strucures (ignores any errors).
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Archive object comparison
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Returns 1 if difference detected
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# 0 if no difference detected
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -1 on error
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.getName()
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal ArchTmpDir2 = tmpDir2 + os.path.basename(base) + t.getName()
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Be optimistic and first try a straight file compare
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # as it will allow us to finish up quickly.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("compareArchives: makedir failed %s" % e)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("compareArchives: makedir failed %s" % e)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # copy over the objects to the temp areas, and
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # unpack them
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal baseCmd = "cp -fp " + base + " " + ArchTmpDir1
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal ptchCmd = "cp -fp " + ptch + " " + ArchTmpDir2
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal baseCmd = "cd " + ArchTmpDir1 + "; " + "jar xf " + bname + \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal ptchCmd = "cd " + ArchTmpDir2 + "; " + "jar xf " + bname + \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal baseCmd = "cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal ptchCmd = "cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Trim leading path off base/ptch file lists
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal flist.append(str_prefix_trunc(fn, ArchTmpDir1))
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal flist.append(str_prefix_trunc(fn, ArchTmpDir2))
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal differs = compareOneFile((ArchTmpDir1 + fn), \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# (Basic) file comparison
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# There's some special case code here for Javadoc HTML files
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Returns 1 if difference detected
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# 0 if no difference detected
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -1 on error
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe if quiet and os.stat(base)[ST_SIZE] != os.stat(ptch)[ST_SIZE] :
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe if len(baseData) != len(ptchData) or baseData != ptchData :
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal diffs = diffData(base, ptch, baseData, ptchData)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Compare two objects by producing a data dump from
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# each object, and then comparing the dump data
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Returns: 1 if a difference is detected
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# 0 if no difference detected
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# -1 upon error
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal tmpFile1 = tmpDir1 + os.path.basename(base) + t.getName()
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName()
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal " | egrep -v '(LINTOBJ|LINTMOD):'" + \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal " | grep -v PASS[1-3]:" + \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal " | egrep -v '(LINTOBJ|LINTMOD):'" + \
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal " | grep -v PASS[1-3]:" + \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe baseCmd = "echo .dump | " + sqlite_cmd + base + " > " + \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe ptchCmd = "echo .dump | " + sqlite_cmd + ptch + " > " + \
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe if len(baseData) != len(ptchData) or baseData != ptchData :
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("compareByDumping: unlink failed %s" % e)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("compareByDumping: unlink failed %s" % e)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Remove the temporary files now.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("compareByDumping: unlink failed %s" % e)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("compareByDumping: unlink failed %s" % e)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# SIGINT signal handler. Changes thread control variable to tell the threads
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# to finish their current job and exit.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal print >> sys.stderr, "Caught Ctrl-C, stopping the threads"
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal# worker thread for changedFiles processing
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # grab the lock to changedFiles and remove one member
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # and process it
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # there is nothing more to do
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Compare two objects. Detect type changes.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Vector off to the appropriate type specific
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# compare routine based on the type.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Verify the file types.
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # If they are different, indicate this and move on
619b4598e0f434792bdadf91cc947739dc734758rotondo difference(fileName, "file type", btype + " to " + ptype)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe elif (fileType == 'Java Archive' or fileType == 'ELF Object Archive') :
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # it has to be some variety of text file
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe# Cleanup and self-terminate
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Performing cleanup (" + str(ret) + ")")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Log file handle
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Globals relating to command line options
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Named temporary files / directories
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Command paths
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe global lintdump_cmd, elfdump_cmd, dump_cmd, dis_cmd, od_cmd, diff_cmd, sqlite_cmd
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Default search path
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Essentially "uname -p"
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # changed files for worker thread processing
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Sort the list of files from a temporary file
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Debugging indicator
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Some globals need to be initialized
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debugon = logging = vdiffs = reportAllSects = sorted = False
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Process command line arguments
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Return values are returned from args() in alpha order
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # (Yes, python functions can return multiple values (ewww))
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Note that args() also set the globals:
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # logging to True if verbose logging (to a file) was enabled
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # vdiffs to True if logged differences aren't to be truncated
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # reportAllSects to True if all ELF section differences are to be reported
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe baseRoot, fileNamesFile, localTools, ptchRoot, results = args()
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Set up the results/log file
ccac5ae384bd76b9eb3473b622170555228b5c5dJosef 'Jeff' Sipek dateTimeStr= "# %04d-%02d-%02d at %02d:%02d:%02d" % time.localtime()[:6]
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Changed files (used only for the sorted case)
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Build paths to the tools required tools
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Try to look for tools in $SRC/tools if the "-t" option
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # was specified
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe error("-t specified, but $SRC not set. Cannot find $SRC/tools")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe wsdiff_path.insert(0, src + "/tools/proto/opt/onbld/bin")
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Set resource limit for number of open files as high as possible.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # This might get handy with big number of threads.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal (nofile_soft, nofile_hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal error("cannot set resource limits for number of open files")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # validate the base and patch paths
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe error("old proto area: " + baseRoot + " does not exist")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe " does not exist")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # log some information identifying the run
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Set up the temporary directories / files
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Could use python's tmpdir routines, but these should
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # be easier to identify / keep around for debugging
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Derive a catalog of new, deleted, and to-be-compared objects
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # either from the specified base and patch proto areas, or from
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # from an input file list
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("The list will appear after the processing is done")
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Here's where all the heavy lifting happens
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # Perform a comparison on each object appearing in
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # both proto areas. compareOneFile will examine the
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # file types of each object, and will vector off to
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # the appropriate comparison routine, where the compare
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # will happen, and any differences will be reported / logged
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # determine maximum number of worker threads by using
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # DMAKE_MAX_JOBS environment variable set by nightly(1)
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # or get number of CPUs in the system
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal max_threads = int(os.environ['DMAKE_MAX_JOBS'])
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal max_threads = os.sysconf("SC_NPROCESSORS_ONLN")
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # If we cannot get number of online CPUs in the system
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # run unparallelized otherwise bump the number up 20%
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # to achieve best results.
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Set signal handler to attempt graceful exit
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal signal.signal( signal.SIGINT, discontinue_processing )
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Create and unleash the threads
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Only at most max_threads must be running at any moment
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal debug("Spawning " + str(max_threads) + " threads");
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Wait for the threads to finish and do cleanup if interrupted
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal if not True in [thread.isAlive() for thread in mythreads]:
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Some threads are still going
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # Interrupted by SIGINT
598cc7dfe526c1fa47f03934c5c6fa6508961296Vladimir Kotal # If the list of differences was sorted it is stored in an array
96ccc8cb05e8790fcc24931dcb155b76e810295cesaxe # We're done, cleanup.