#!/bin/bash
|
#!/bin/bash
|
# Copyright (C) 2006, 2008 Free Software Foundation
|
# Copyright (C) 2006, 2008 Free Software Foundation
|
#
|
#
|
# Analyze changes in GCC DejaGNU test logs for binutils, gcc, gdb, etc.
|
# Analyze changes in GCC DejaGNU test logs for binutils, gcc, gdb, etc.
|
# Original version written in 2005 by James Lemke <jwlemke@wasabisystems.com>.
|
# Original version written in 2005 by James Lemke <jwlemke@wasabisystems.com>.
|
#
|
#
|
# See usage() below.
|
# See usage() below.
|
|
|
usage () {
|
usage () {
|
cat <<EOF >&2
|
cat <<EOF >&2
|
Usage:
|
Usage:
|
dg-cmp-results.sh [-v] [-v] [-v] <variant-name> <old-file> <new-file>
|
dg-cmp-results.sh [-v] [-v] [-v] <variant-name> <old-file> <new-file>
|
<variant-name> names the desired variant, "/" must be written as "\/".
|
<variant-name> names the desired variant, "/" must be written as "\/".
|
Use the empty string ("") for the first variant in each file.
|
Use the empty string ("") for the first variant in each file.
|
Output is to stdout.
|
Output is to stdout.
|
Non-verbose output is degradation info like PASS->FAIL.
|
Non-verbose output is degradation info like PASS->FAIL.
|
-v adds improvement info like FAIL->PASS.
|
-v adds improvement info like FAIL->PASS.
|
-v -v adds info like tests that are no longer run.
|
-v -v adds info like tests that are no longer run.
|
-v -v -v adds info for tests that have not changed status.
|
-v -v -v adds info for tests that have not changed status.
|
-v -v -v -v is used for debugging.
|
-v -v -v -v is used for debugging.
|
EOF
|
EOF
|
}
|
}
|
|
|
verbose=0
|
verbose=0
|
while test "$1" = "-v"; do
|
while test "$1" = "-v"; do
|
verbose=`expr $verbose + 1`
|
verbose=`expr $verbose + 1`
|
shift
|
shift
|
done
|
done
|
|
|
if test $# -ne 3 ; then
|
if test $# -ne 3 ; then
|
usage
|
usage
|
exit 1
|
exit 1
|
fi
|
fi
|
|
|
if test ! -f "$2"; then
|
if test ! -f "$2"; then
|
echo "unable to open $2" >&2
|
echo "unable to open $2" >&2
|
exit 1
|
exit 1
|
fi
|
fi
|
|
|
if test ! -f "$3"; then
|
if test ! -f "$3"; then
|
echo "unable to open $3" >&2
|
echo "unable to open $3" >&2
|
exit 1
|
exit 1
|
fi
|
fi
|
|
|
# Command differences for various platforms.
|
# Command differences for various platforms.
|
case `uname -s` in
|
case `uname -s` in
|
Darwin|NetBSD)
|
Darwin|NetBSD)
|
E=-E # sed
|
E=-E # sed
|
;;
|
;;
|
*)
|
*)
|
E=-r # sed
|
E=-r # sed
|
;;
|
;;
|
esac
|
esac
|
|
|
# sections are identified by separator lines beginning with '\t\t==='.
|
# sections are identified by separator lines beginning with '\t\t==='.
|
# section 0 identifies run date, target, and host.
|
# section 0 identifies run date, target, and host.
|
# section 1 and subsequent contain test data for a target variant.
|
# section 1 and subsequent contain test data for a target variant.
|
# -skip to /^Running target/ and use that line to identify the variant.
|
# -skip to /^Running target/ and use that line to identify the variant.
|
# -subsequent lines contain the result data. They begin with:
|
# -subsequent lines contain the result data. They begin with:
|
# '(PASS|FAIL|XFAIL|XPASS|UNTESTED|UNSUPPORTED|UNRESOLVED):'
|
# '(PASS|FAIL|XFAIL|XPASS|UNTESTED|UNSUPPORTED|UNRESOLVED):'
|
VARIANT="$1"
|
VARIANT="$1"
|
OFILE="$2"
|
OFILE="$2"
|
OBASE=`basename "$2"`
|
OBASE=`basename "$2"`
|
NFILE="$3"
|
NFILE="$3"
|
NBASE=`basename "$3"`
|
NBASE=`basename "$3"`
|
|
|
echo "dg-cmp-results.sh: Verbosity is ${verbose}, Variant is \"${VARIANT}\""
|
echo "dg-cmp-results.sh: Verbosity is ${verbose}, Variant is \"${VARIANT}\""
|
echo
|
echo
|
|
|
header="^Running target $VARIANT"
|
header="^Running target $VARIANT"
|
|
|
temp=`grep "$header" $OFILE`
|
temp=`grep "$header" $OFILE`
|
if test -z "$temp"; then
|
if test -z "$temp"; then
|
echo "Error: variant \"$VARIANT\" not found in $OFILE."
|
echo "Error: variant \"$VARIANT\" not found in $OFILE."
|
exit 1
|
exit 1
|
fi
|
fi
|
temp=`grep "$header" $NFILE`
|
temp=`grep "$header" $NFILE`
|
if test -z "$temp"; then
|
if test -z "$temp"; then
|
echo "Error: variant \"$VARIANT\" not found in $NFILE."
|
echo "Error: variant \"$VARIANT\" not found in $NFILE."
|
exit 1
|
exit 1
|
fi
|
fi
|
unset temp
|
unset temp
|
|
|
# Copy out the old file's section 0.
|
# Copy out the old file's section 0.
|
echo "Older log file: $OFILE"
|
echo "Older log file: $OFILE"
|
sed $E -e '/^[[:space:]]+===/,$d' $OFILE
|
sed $E -e '/^[[:space:]]+===/,$d' $OFILE
|
|
|
# Copy out the new file's section 0.
|
# Copy out the new file's section 0.
|
echo "Newer log file: $NFILE"
|
echo "Newer log file: $NFILE"
|
sed $E -e '/^[[:space:]]+===/,$d' $NFILE
|
sed $E -e '/^[[:space:]]+===/,$d' $NFILE
|
|
|
# Create a temporary file from the old file's interesting section.
|
# Create a temporary file from the old file's interesting section.
|
sed $E -e "1,/$header/d" \
|
sed $E -e "1,/$header/d" \
|
-e '/^[[:space:]]+===/,$d' \
|
-e '/^[[:space:]]+===/,$d' \
|
-e '/^[A-Z]+:/!d' \
|
-e '/^[A-Z]+:/!d' \
|
-e '/^(WARNING|ERROR):/d' \
|
-e '/^(WARNING|ERROR):/d' \
|
-e 's/\r$//' \
|
-e 's/\r$//' \
|
-e 's/^/O:/' \
|
-e 's/^/O:/' \
|
$OFILE |
|
$OFILE |
|
sort -s -t : -k 3b - \
|
sort -s -t : -k 3b - \
|
>/tmp/o$$-$OBASE
|
>/tmp/o$$-$OBASE
|
|
|
# Create a temporary file from the new file's interesting section.
|
# Create a temporary file from the new file's interesting section.
|
sed $E -e "1,/$header/d" \
|
sed $E -e "1,/$header/d" \
|
-e '/^[[:space:]]+===/,$d' \
|
-e '/^[[:space:]]+===/,$d' \
|
-e '/^[A-Z]+:/!d' \
|
-e '/^[A-Z]+:/!d' \
|
-e '/^(WARNING|ERROR):/d' \
|
-e '/^(WARNING|ERROR):/d' \
|
-e 's/\r$//' \
|
-e 's/\r$//' \
|
-e 's/^/N:/' \
|
-e 's/^/N:/' \
|
$NFILE |
|
$NFILE |
|
sort -s -t : -k 3b - \
|
sort -s -t : -k 3b - \
|
>/tmp/n$$-$NBASE
|
>/tmp/n$$-$NBASE
|
|
|
# Merge the two files, then compare adjacent lines.
|
# Merge the two files, then compare adjacent lines.
|
# Comparison is complicated by tests that may be run multiple times.
|
# Comparison is complicated by tests that may be run multiple times.
|
# If that case, we assume that the order is the same in both files.
|
# If that case, we assume that the order is the same in both files.
|
cat <<EOF >compare-$$.awk
|
cat <<EOF >compare-$$.awk
|
BEGIN {
|
BEGIN {
|
FS = ":"
|
FS = ":"
|
queue1 = 1; queueN = 0; status[queue1] = ""; name[queue1] = ""
|
queue1 = 1; queueN = 0; status[queue1] = ""; name[queue1] = ""
|
verbose = verbose + 0 # Make sure it's defined.
|
verbose = verbose + 0 # Make sure it's defined.
|
}
|
}
|
|
|
# FIFO circular queue
|
# FIFO circular queue
|
function push(st, nm) {
|
function push(st, nm) {
|
queueN += 1; status[queueN] = st; name[queueN] = nm
|
queueN += 1; status[queueN] = st; name[queueN] = nm
|
}
|
}
|
function peek() {
|
function peek() {
|
result = 0
|
result = 0
|
if (queueN >= queue1) result = queue1
|
if (queueN >= queue1) result = queue1
|
return result
|
return result
|
}
|
}
|
function drop() {
|
function drop() {
|
queue1 += 1
|
queue1 += 1
|
if (queue1 > queueN) { queue1 = 1; queueN = 0; }
|
if (queue1 > queueN) { queue1 = 1; queueN = 0; }
|
}
|
}
|
|
|
function compare(st, nm) {
|
function compare(st, nm) {
|
old = peek()
|
old = peek()
|
if (old == 0) {
|
if (old == 0) {
|
# This new test wasn't run last time.
|
# This new test wasn't run last time.
|
if (verbose >= 2) printf("NA->%s:%s\n", st, nm)
|
if (verbose >= 2) printf("NA->%s:%s\n", st, nm)
|
}
|
}
|
else {
|
else {
|
# Compare this new test to the first queued old one.
|
# Compare this new test to the first queued old one.
|
if (verbose >= 4) {
|
if (verbose >= 4) {
|
printf("Comparing two lines:\n O:%s:%s\n N:%s:%s\n",
|
printf("Comparing two lines:\n O:%s:%s\n N:%s:%s\n",
|
status[old], name[old], st, nm)
|
status[old], name[old], st, nm)
|
}
|
}
|
if (name[old] != nm) {
|
if (name[old] != nm) {
|
# The old test wasn't run this time and
|
# The old test wasn't run this time and
|
# the new test wasn't run last time.
|
# the new test wasn't run last time.
|
if (verbose >= 2) {
|
if (verbose >= 2) {
|
printf("%s->NA:%s\n", status[old], name[old])
|
printf("%s->NA:%s\n", status[old], name[old])
|
if (nm != "") printf("NA->%s:%s\n", st, nm)
|
if (nm != "") printf("NA->%s:%s\n", st, nm)
|
}
|
}
|
drop()
|
drop()
|
}
|
}
|
else {
|
else {
|
notable = 0
|
notable = 0
|
if (status[old] == st) {
|
if (status[old] == st) {
|
# Status of this test has not changed.
|
# Status of this test has not changed.
|
if (verbose >= 3) printf("%s:%s\n", st, nm)
|
if (verbose >= 3) printf("%s:%s\n", st, nm)
|
}
|
}
|
else if(status[old] == "PASS" && st == "XFAIL") {
|
else if(status[old] == "PASS" && st == "XFAIL") {
|
if (verbose >= 1) notable = 1
|
if (verbose >= 1) notable = 1
|
}
|
}
|
else if(status[old] == "PASS" || st == "FAIL") {
|
else if(status[old] == "PASS" || st == "FAIL") {
|
# Test did pass but doesn't now
|
# Test did pass but doesn't now
|
# or didn't fail but does now.
|
# or didn't fail but does now.
|
notable = 1
|
notable = 1
|
}
|
}
|
else if(st == "PASS") {
|
else if(st == "PASS") {
|
# Test didn't pass but does now.
|
# Test didn't pass but does now.
|
if (verbose >= 1) notable = 1
|
if (verbose >= 1) notable = 1
|
}
|
}
|
else if(verbose >= 2) {
|
else if(verbose >= 2) {
|
# Miscellaneous status change.
|
# Miscellaneous status change.
|
notable = 1
|
notable = 1
|
}
|
}
|
if (notable > 0) printf("%s->%s:%s\n", status[old], st, nm)
|
if (notable > 0) printf("%s->%s:%s\n", status[old], st, nm)
|
drop()
|
drop()
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
/^O:/ {
|
/^O:/ {
|
while (old = peek()) {
|
while (old = peek()) {
|
if (name[old] == \$3) break;
|
if (name[old] == \$3) break;
|
# The queued test is no longer run.
|
# The queued test is no longer run.
|
compare("", "");
|
compare("", "");
|
}
|
}
|
# Save this test for later comparison.
|
# Save this test for later comparison.
|
push(\$2, \$3)
|
push(\$2, \$3)
|
}
|
}
|
|
|
/^N:/ {
|
/^N:/ {
|
compare(\$2, \$3)
|
compare(\$2, \$3)
|
}
|
}
|
|
|
END {
|
END {
|
while (old = peek()) compare("", "")
|
while (old = peek()) compare("", "")
|
}
|
}
|
EOF
|
EOF
|
sort -m -s -t : -k 3b /tmp/o$$-$OBASE /tmp/n$$-$NBASE |
|
sort -m -s -t : -k 3b /tmp/o$$-$OBASE /tmp/n$$-$NBASE |
|
awk -v verbose=$verbose -f compare-$$.awk /dev/stdin
|
awk -v verbose=$verbose -f compare-$$.awk /dev/stdin
|
|
|
# Delete the temporary files.
|
# Delete the temporary files.
|
rm -f compare-$$.awk /tmp/o$$-$OBASE /tmp/n$$-$NBASE
|
rm -f compare-$$.awk /tmp/o$$-$OBASE /tmp/n$$-$NBASE
|
|
|
exit 0
|
exit 0
|
|
|
