4bd025742d
Right now only non-SMT SE regression tests have been added back in. The rest are pending getting SMT working, and consolidating the FS configuration files.
Eventually support for different OSs can be added so you can specify which versions of the binary you want to run from one config file.
Note: mp-test1 doesn't have any reference stats because MP mode doesn't currently work. The test itself should probably work once the code is fixed.
SConstruct:
Updates to allow for regression tests to work via the command line "scons build/ALPHA_SE/test/debug/quick" and such once again.
src/cpu/SConscript:
Keep a list of SMT supporting CPUs so that the regression tests can easily specify which CPUs to use if they are SMT only.
--HG--
extra : convert_revision : 34e6286150aae8f316ae694f6c00be8f510522f2
409 lines
10 KiB
Perl
Executable file
409 lines
10 KiB
Perl
Executable file
#!/usr/bin/perl
|
|
# Copyright (c) 2001-2005 The Regents of The University of Michigan
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met: redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer;
|
|
# redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution;
|
|
# neither the name of the copyright holders nor the names of its
|
|
# contributors may be used to endorse or promote products derived from
|
|
# this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
# Authors: Steve Reinhardt
|
|
|
|
#
|
|
# This script diffs two SimpleScalar statistics output files.
|
|
#
|
|
|
|
use Getopt::Std;
|
|
|
|
#
|
|
# -t thresh sets threshold for ignoring differences (in %)
|
|
# -p sorts differences by % chg (default is alphabetic)
|
|
# -f ignores fetch-loss statistics
|
|
# -d ignores all distributions
|
|
#
|
|
|
|
getopts('dfn:pt:h');
|
|
|
|
if ($#ARGV < 1)
|
|
{
|
|
print "\nError: need two file arguments (<reference> <new>).\n";
|
|
print " Options: -d = Ignore distributions\n";
|
|
print " -f = Ignore fetch-loss stats\n";
|
|
print " -p = Sort errors by percentage\n";
|
|
print " -h = Diff header info separately from stats\n";
|
|
print " -n <num> = Print top <num> errors (default 20)\n";
|
|
print " -t <num> = Error threshold in percent (default 1)\n\n";
|
|
die -1;
|
|
}
|
|
|
|
open(REF, "<$ARGV[0]") or die "Error: can't open $ARGV[0].\n";
|
|
open(NEW, "<$ARGV[1]") or die "Error: can't open $ARGV[1].\n";
|
|
|
|
|
|
#
|
|
# Things that really should be adjustable via the command line
|
|
#
|
|
|
|
# Ignorable error (in percent)
|
|
$err_thresh = ($opt_t) ? $opt_t : 0;
|
|
|
|
# Number of stats to print before omitting
|
|
$omit_count = ($opt_n) ? $opt_n : 20;
|
|
|
|
|
|
#
|
|
# First copy everything up to the simulation statistics to a pair of
|
|
# temporary files, stripping out date-related items, and do a plain
|
|
# diff. Any differences in the arguments are not necessarily an issue;
|
|
# any differences in the program output should be caught by the EIO
|
|
# mechanism if an EIO file is used.
|
|
#
|
|
|
|
# copy_header takes input filehandle and output filename
|
|
|
|
sub copy_header
|
|
{
|
|
my ($inhandle, $outname) = @_;
|
|
|
|
open(OUTPUT, ">$outname") or die "Error: can't open $outname.\n";
|
|
|
|
while (<$inhandle>)
|
|
{
|
|
# strip out lines that can vary
|
|
next if /^(command line:|M5 compiled on |M5 simulation started |M5 executing on )/;
|
|
last if /Begin Simulation Statistics/;
|
|
print OUTPUT;
|
|
}
|
|
close OUTPUT;
|
|
}
|
|
|
|
if ($opt_h) {
|
|
|
|
# Diff header separately from stats
|
|
|
|
$refheader = "/tmp/smt-test.refheader.$$";
|
|
$newheader = "/tmp/smt-test.newheader.$$";
|
|
|
|
copy_header(\*REF, $refheader);
|
|
copy_header(\*NEW, $newheader);
|
|
|
|
print "\n===== Header and program output differences =====\n\n";
|
|
|
|
print `diff $refheader $newheader`;
|
|
|
|
print "\n===== Statistics differences =====\n\n";
|
|
}
|
|
|
|
#
|
|
# Now parse statistics
|
|
#
|
|
|
|
#
|
|
# This function takes an open filehandle and returns a reference to
|
|
# a hash containing all the statistics variables and their values.
|
|
#
|
|
sub parse_file
|
|
{
|
|
$stathandle = shift;
|
|
|
|
$in_dist = undef;
|
|
$hashref = { }; # initialize hash for values
|
|
|
|
while (<$stathandle>)
|
|
{
|
|
next if /^\s*$/; # skip blank lines
|
|
next if /^\*\*Ignore/; # temporary, to make totaling scripts easy for ISCA 03
|
|
last if /End Simulation Statistics/;
|
|
|
|
s/ *#.*//; # strip comments
|
|
|
|
if (/^Memory usage: (\d+) KBytes/) {
|
|
$stat = 'memory usage';
|
|
$value = $1;
|
|
}
|
|
elsif ($in_dist) {
|
|
if ($in_dist =~ /^fetch_loss_counters/) {
|
|
if (/^fetch_loss_counters_\d+\.end/) {
|
|
# end line of distribution: clear $in_dist flag
|
|
$in_dist = undef;
|
|
next;
|
|
}
|
|
else {
|
|
next if $opt_f;
|
|
|
|
($stat, $value) = /^(\S+)\s+(.*)/;
|
|
}
|
|
}
|
|
else {
|
|
if (/(.*)\.end_dist/) {
|
|
# end line of distribution: clear $in_dist flag
|
|
$in_dist = undef;
|
|
next;
|
|
}
|
|
if ($opt_d) {
|
|
next; # bail out if we are ignoring dists...
|
|
}
|
|
elsif (/(.*)\.(min|max)_value/) {
|
|
# treat these like normal stats
|
|
($stat, $value) = /^(\S+)\s+(.*)/;
|
|
}
|
|
else {
|
|
# this is ugly because labels in the distribution
|
|
# buckets don't start in column 0 and may include
|
|
# embedded spaces
|
|
($stat, $value) =
|
|
/^\s*(\S+(?:.*\S)?)\s+(\d+)\s+\d+\.\d+%/;
|
|
$stat = $in_dist . '::' . $stat;
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
if (/(.*)\.start_dist/) {
|
|
# start line of distribution: set $in_dist flag
|
|
# and save distribution name for future reference
|
|
$in_dist = $1;
|
|
$stat = $1;
|
|
$value = 0;
|
|
}
|
|
elsif (/^(fetch_loss_counters_\d+)\.start/) {
|
|
# treat fetch loss counters like distribution, sort of
|
|
$in_dist = $1;
|
|
$stat = $1;
|
|
$value = 0;
|
|
}
|
|
else {
|
|
($stat, $value) = /^(\S+)\s+(.*)/;
|
|
}
|
|
}
|
|
|
|
$$hashref{$stat} = $value;
|
|
}
|
|
|
|
close($stathandle);
|
|
return $hashref;
|
|
}
|
|
|
|
|
|
#
|
|
# pct_diff($old, $new) returns percent difference from $old to $new.
|
|
#
|
|
sub pct_diff
|
|
{
|
|
my ($old, $new) = @_;
|
|
return ($old == 0) ? (($new == 0) ? 0 : 9999) : 100 * ($new - $old) / $old;
|
|
}
|
|
|
|
|
|
#
|
|
# Statistics to ignore: these relate to simulator performance, not
|
|
# correctness, so don't fail on changes here.
|
|
#
|
|
%ignore = (
|
|
'host_seconds' => 1,
|
|
'host_tick_rate' => 1,
|
|
'host_inst_rate' => 1,
|
|
'host_mem_usage' => 1
|
|
);
|
|
|
|
#
|
|
# List of key statistics (always displayed)
|
|
# ==> list stats here WITHOUT trailing thread ID
|
|
#
|
|
@key_stat_list = (
|
|
'COM:IPC',
|
|
'ISSUE:MSIPC',
|
|
'COM:count',
|
|
'host_inst_rate',
|
|
'sim_insts',
|
|
'sim_ticks',
|
|
'host_mem_usage'
|
|
);
|
|
|
|
$key_stat_pattern = join('|', @key_stat_list);
|
|
|
|
# initialize first statistics from each file
|
|
|
|
$max_err_mag = 0;
|
|
|
|
$refhash = parse_file(\*REF);
|
|
$newhash = parse_file(\*NEW);
|
|
|
|
# The string sim-smt prints on a divide by zero
|
|
$divbyzero = '<err: divide by zero>';
|
|
|
|
foreach $stat (sort keys %$refhash)
|
|
{
|
|
$refvalue = $$refhash{$stat};
|
|
$newvalue = $$newhash{$stat};
|
|
|
|
if (!defined($newvalue)) {
|
|
# stat missing from new file
|
|
push @missing_stats, $stat;
|
|
next;
|
|
}
|
|
|
|
if ($stat =~ /($key_stat_pattern)/o) {
|
|
# key statistics: always record & display changes in these
|
|
push @key_stats, [$stat, $refvalue, $newvalue];
|
|
}
|
|
|
|
if ($ignore{$stat} or $refvalue eq $newvalue) {
|
|
# stat is in "ignore" list, or hasn't changed
|
|
}
|
|
else {
|
|
if ($refvalue eq $divbyzero || $newvalue eq $divbyzero) {
|
|
# one or the other was a divide by zero:
|
|
# no point in trying to quantify error
|
|
print "$stat: $refvalue --> $newvalue\n";
|
|
}
|
|
else {
|
|
$reldiff = pct_diff($refvalue, $newvalue);
|
|
$diffmag = abs($reldiff);
|
|
|
|
if ($diffmag > $err_thresh) {
|
|
push @errs,
|
|
[$stat, $refvalue, $newvalue, $reldiff];
|
|
}
|
|
|
|
if ($diffmag > $max_err_mag) {
|
|
$max_err_mag = $diffmag;
|
|
}
|
|
}
|
|
}
|
|
|
|
# remove from new hash so we can detect added stats
|
|
delete $$newhash{$stat};
|
|
}
|
|
|
|
|
|
#
|
|
# All done. Print comparison summary.
|
|
#
|
|
|
|
printf("Maximum error magnitude: %+f%%\n\n", $max_err_mag);
|
|
|
|
printf(" %-30s %10s %10s %10s %7s\n", ' ', 'Reference', 'New Value', 'Abs Diff', 'Pct Chg');
|
|
|
|
printf("Key statistics:\n\n");
|
|
|
|
foreach $key_stat (@key_stats)
|
|
{
|
|
($statname, $refvalue, $newvalue, $reldiff) = @$key_stat;
|
|
|
|
# deduce format from reference value
|
|
$pointpos = rindex($refvalue, '.');
|
|
$digits = ($pointpos < 0) ? 0 :(length($refvalue) - $pointpos - 1);
|
|
$fmt = "%10.${digits}f";
|
|
|
|
# print differing values with absolute and relative error
|
|
printf(" %-30s $fmt $fmt $fmt %+7.2f%%\n",
|
|
$statname, $refvalue, $newvalue,
|
|
$newvalue - $refvalue, pct_diff($refvalue, $newvalue));
|
|
}
|
|
|
|
printf("\nLargest $omit_count relative errors (> %d%%):\n\n", $err_thresh);
|
|
|
|
$num_errs = 0;
|
|
|
|
if ($opt_p)
|
|
{
|
|
# sort differences by percent change
|
|
@errs = sort { abs($$b[3]) <=> abs($$a[3]) } @errs;
|
|
}
|
|
|
|
foreach $err (@errs)
|
|
{
|
|
($statname, $refvalue, $newvalue, $reldiff) = @$err;
|
|
|
|
# deduce format from reference value
|
|
$pointpos1 = rindex($refvalue, '.');
|
|
$digits1 = ($pointpos1 < 0) ? 0 :(length($refvalue) - $pointpos1 - 1);
|
|
$pointpos2 = rindex($newvalue, '.');
|
|
$digits2 = ($pointpos2 < 0) ? 0 :(length($newvalue) - $pointpos2 - 1);
|
|
$digits = ($digits1 > $digits2) ? $digits1 : $digits2;
|
|
$fmt = "%10.${digits}f";
|
|
|
|
# print differing values with absolute and relative error
|
|
printf(" %-30s $fmt $fmt $fmt %+7.2f%%\n",
|
|
$statname, $refvalue, $newvalue, $newvalue - $refvalue, $reldiff);
|
|
|
|
# only print top N errors
|
|
if (++$num_errs >= $omit_count)
|
|
{
|
|
print "[... additional errors omitted ...]\n";
|
|
last;
|
|
}
|
|
}
|
|
|
|
#
|
|
# Report missing stats, but first filter out distribution buckets:
|
|
# these are mostly noise
|
|
|
|
@missing_stats = grep { !/::(\d+|overflows)?$/ } @missing_stats;
|
|
|
|
# get count
|
|
$missing_stats = scalar(@missing_stats);
|
|
|
|
if ($missing_stats)
|
|
{
|
|
print "\nMissing $missing_stats reference statistics:\n\n";
|
|
foreach $stat (@missing_stats)
|
|
{
|
|
# print "\t$stat\n";
|
|
printf " %-50s ", $stat;
|
|
print "$$refhash{$stat}\n";
|
|
}
|
|
}
|
|
|
|
#
|
|
# Any stats left in newhash are added since the reference file
|
|
#
|
|
|
|
@added_stats = keys %$newhash;
|
|
|
|
# first filter out distribution buckets: mostly noise
|
|
|
|
@added_stats = grep { !/::(\d+|overflows)?$/ } @added_stats;
|
|
|
|
# get count
|
|
$added_stats = scalar(@added_stats);
|
|
|
|
if ($added_stats)
|
|
{
|
|
print "\nFound $added_stats new statistics:\n\n";
|
|
foreach $stat (sort @added_stats)
|
|
{
|
|
# print "\t$stat\n";
|
|
printf " %-50s ", $stat;
|
|
print "$$newhash{$stat}\n";
|
|
}
|
|
}
|
|
|
|
cleanup();
|
|
# Exit code is 0 if no stats error, 1 otherwise
|
|
$status = ($max_err_mag == 0.0) ? 0 : 1;
|
|
exit $status;
|
|
|
|
sub cleanup
|
|
{
|
|
unlink($refheader) if ($refheader);
|
|
unlink($newheader) if ($newheader);
|
|
}
|