scons: Identify runs that fail and runs with stats differences
This patch changes the regression script such that it is possible to identify the runs that fail with an exit code, and those that finish with stats differences. The ones that truly fail are reported as FAILED, and those that finish with changed stats as CHANGED. The yellow colour has been reclaimed from the skipped regressions and is now used for the changed ones. With no obvious good option left the skipped ones are now in cyan. While I was editing the script I also bumped any occurence of M5 to gem5.
This commit is contained in:
parent
997a6a4add
commit
718fb40709
1 changed files with 22 additions and 17 deletions
|
@ -82,7 +82,7 @@ def run_test(target, source, env):
|
||||||
target[0] : status
|
target[0] : status
|
||||||
|
|
||||||
Sources are:
|
Sources are:
|
||||||
source[0] : M5 binary
|
source[0] : gem5 binary
|
||||||
source[1] : tests/run.py script
|
source[1] : tests/run.py script
|
||||||
source[2] : reference stats file
|
source[2] : reference stats file
|
||||||
|
|
||||||
|
@ -105,10 +105,14 @@ def run_test(target, source, env):
|
||||||
if env['BATCH']:
|
if env['BATCH']:
|
||||||
cmd = '%s -t %d %s' % (env['BATCH_CMD'], timeout, cmd)
|
cmd = '%s -t %d %s' % (env['BATCH_CMD'], timeout, cmd)
|
||||||
|
|
||||||
|
# Create a default value for the status string, changed as needed
|
||||||
|
# based on the status.
|
||||||
|
status_str = "passed."
|
||||||
|
|
||||||
pre_exec_time = time.time()
|
pre_exec_time = time.time()
|
||||||
status = env.Execute(env.subst(cmd, target=target, source=source))
|
status = env.Execute(env.subst(cmd, target=target, source=source))
|
||||||
if status == 0:
|
if status == 0:
|
||||||
# M5 terminated normally.
|
# gem5 terminated normally.
|
||||||
# Run diff on output & ref directories to find differences.
|
# Run diff on output & ref directories to find differences.
|
||||||
# Exclude the stats file since we will use diff-out on that.
|
# Exclude the stats file since we will use diff-out on that.
|
||||||
|
|
||||||
|
@ -137,35 +141,34 @@ def run_test(target, source, env):
|
||||||
diffcmd = '$DIFFOUT ${SOURCES[2]} %s > %s' \
|
diffcmd = '$DIFFOUT ${SOURCES[2]} %s > %s' \
|
||||||
% (os.path.join(tgt_dir, 'stats.txt'), statsdiff)
|
% (os.path.join(tgt_dir, 'stats.txt'), statsdiff)
|
||||||
diffcmd = env.subst(diffcmd, target=target, source=source)
|
diffcmd = env.subst(diffcmd, target=target, source=source)
|
||||||
status = env.Execute(diffcmd, strfunction=None)
|
diff_status = env.Execute(diffcmd, strfunction=None)
|
||||||
|
# If there is a difference, change the status string to say so
|
||||||
|
if diff_status != 0:
|
||||||
|
status_str = "CHANGED!"
|
||||||
print "===== Statistics differences ====="
|
print "===== Statistics differences ====="
|
||||||
print contents(statsdiff)
|
print contents(statsdiff)
|
||||||
|
|
||||||
else: # m5 exit status != 0
|
else: # gem5 exit status != 0
|
||||||
# M5 did not terminate properly, so no need to check the output
|
# Consider it a failed test unless the exit status is 2
|
||||||
|
status_str = "FAILED!"
|
||||||
|
# gem5 did not terminate properly, so no need to check the output
|
||||||
if signaled(status):
|
if signaled(status):
|
||||||
print 'M5 terminated with signal', signum(status)
|
print 'gem5 terminated with signal', signum(status)
|
||||||
if signum(status) in retry_signals:
|
if signum(status) in retry_signals:
|
||||||
# Consider the test incomplete; don't create a 'status' output.
|
# Consider the test incomplete; don't create a 'status' output.
|
||||||
# Hand the return status to scons and let scons decide what
|
# Hand the return status to scons and let scons decide what
|
||||||
# to do about it (typically terminate unless run with -k).
|
# to do about it (typically terminate unless run with -k).
|
||||||
return status
|
return status
|
||||||
elif status == 2:
|
elif status == 2:
|
||||||
# The test was skipped
|
# The test was skipped, change the status string to say so
|
||||||
pass
|
status_str = "skipped."
|
||||||
else:
|
else:
|
||||||
print 'M5 exited with non-zero status', status
|
print 'gem5 exited with non-zero status', status
|
||||||
# complete but failed execution (call to exit() with non-zero
|
# complete but failed execution (call to exit() with non-zero
|
||||||
# status, SIGABORT due to assertion failure, etc.)... fall through
|
# status, SIGABORT due to assertion failure, etc.)... fall through
|
||||||
# and generate FAILED status as if output comparison had failed
|
# and generate FAILED status as if output comparison had failed
|
||||||
|
|
||||||
# Generate status file contents based on exit status of m5 or diff-out
|
# Generate status file contents based on exit status of gem5 and diff-out
|
||||||
if status == 0:
|
|
||||||
status_str = "passed."
|
|
||||||
elif status == 2:
|
|
||||||
status_str = "skipped."
|
|
||||||
else:
|
|
||||||
status_str = "FAILED!"
|
|
||||||
f = file(str(target[0]), 'w')
|
f = file(str(target[0]), 'w')
|
||||||
print >>f, tgt_dir, status_str
|
print >>f, tgt_dir, status_str
|
||||||
f.close()
|
f.close()
|
||||||
|
@ -191,10 +194,12 @@ def print_test(target, source, env):
|
||||||
# skip the punctuation
|
# skip the punctuation
|
||||||
if status == "FAILED!":
|
if status == "FAILED!":
|
||||||
status = termcap.Red + status[:-1] + termcap.Normal + status[-1]
|
status = termcap.Red + status[:-1] + termcap.Normal + status[-1]
|
||||||
|
elif status == "CHANGED!":
|
||||||
|
status = termcap.Yellow + status[:-1] + termcap.Normal + status[-1]
|
||||||
elif status == "passed.":
|
elif status == "passed.":
|
||||||
status = termcap.Green + status[:-1] + termcap.Normal + status[-1]
|
status = termcap.Green + status[:-1] + termcap.Normal + status[-1]
|
||||||
elif status == "skipped.":
|
elif status == "skipped.":
|
||||||
status = termcap.Yellow + status[:-1] + termcap.Normal + status[-1]
|
status = termcap.Cyan + status[:-1] + termcap.Normal + status[-1]
|
||||||
|
|
||||||
# put it back in the list and join with space
|
# put it back in the list and join with space
|
||||||
words[-1] = status
|
words[-1] = status
|
||||||
|
|
Loading…
Reference in a new issue