c20b6c56f6
There are cases where we need to ignore files with specific extensions (e.g., when Mercurial litters the file system with patch rejects). Implement this functionality using a helper class (FileIgnoreList) that supports both regular expressions and basic string comparisons. Change-Id: I34549754bd2e10ed230ffb2dc057403349f8fa78 Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
364 lines
12 KiB
Python
364 lines
12 KiB
Python
#!/usr/bin/env python
|
|
#
|
|
# Copyright (c) 2016 ARM Limited
|
|
# All rights reserved
|
|
#
|
|
# The license below extends only to copyright in the software and shall
|
|
# not be construed as granting a license to any other intellectual
|
|
# property including but not limited to intellectual property relating
|
|
# to a hardware implementation of the functionality of the software
|
|
# licensed hereunder. You may use the software subject to the license
|
|
# terms below provided that you ensure that this notice is replicated
|
|
# unmodified and in its entirety in all distributions of the software,
|
|
# modified or unmodified, in source code or in binary form.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met: redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer;
|
|
# redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution;
|
|
# neither the name of the copyright holders nor the names of its
|
|
# contributors may be used to endorse or promote products derived from
|
|
# this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
# Authors: Andreas Sandberg
|
|
|
|
from abc import ABCMeta, abstractmethod
|
|
import os
|
|
from collections import namedtuple
|
|
from units import *
|
|
from results import TestResult
|
|
import shutil
|
|
|
|
_test_base = os.path.join(os.path.dirname(__file__), "..")
|
|
|
|
ClassicConfig = namedtuple("ClassicConfig", (
|
|
"category",
|
|
"mode",
|
|
"workload",
|
|
"isa",
|
|
"os",
|
|
"config",
|
|
))
|
|
|
|
# There are currently two "classes" of test
|
|
# configurations. Architecture-specific ones and generic ones
|
|
# (typically SE mode tests). In both cases, the configuration name
|
|
# matches a file in tests/configs/ that will be picked up by the test
|
|
# runner (run.py).
|
|
#
|
|
# Architecture specific configurations are listed in the arch_configs
|
|
# dictionary. This is indexed by a (cpu architecture, gpu
|
|
# architecture) tuple. GPU architecture is optional and may be None.
|
|
#
|
|
# Generic configurations are listed in the generic_configs tuple.
|
|
#
|
|
# When discovering available test cases, this script look uses the
|
|
# test list as a list of /candidate/ configurations. A configuration
|
|
# is only used if a test has a reference output for that
|
|
# configuration. In addition to the base configurations from
|
|
# arch_configs and generic_configs, a Ruby configuration may be
|
|
# appended to the base name (this is probed /in addition/ to the
|
|
# original name. See get_tests() for details.
|
|
#
|
|
arch_configs = {
|
|
("alpha", None) : (
|
|
'tsunami-simple-atomic',
|
|
'tsunami-simple-timing',
|
|
'tsunami-simple-atomic-dual',
|
|
'tsunami-simple-timing-dual',
|
|
'twosys-tsunami-simple-atomic',
|
|
'tsunami-o3', 'tsunami-o3-dual',
|
|
'tsunami-minor', 'tsunami-minor-dual',
|
|
'tsunami-switcheroo-full',
|
|
),
|
|
|
|
("arm", None) : (
|
|
'simple-atomic-dummychecker',
|
|
'o3-timing-checker',
|
|
'realview-simple-atomic',
|
|
'realview-simple-atomic-dual',
|
|
'realview-simple-atomic-checkpoint',
|
|
'realview-simple-timing',
|
|
'realview-simple-timing-dual',
|
|
'realview-o3',
|
|
'realview-o3-checker',
|
|
'realview-o3-dual',
|
|
'realview-minor',
|
|
'realview-minor-dual',
|
|
'realview-switcheroo-atomic',
|
|
'realview-switcheroo-timing',
|
|
'realview-switcheroo-o3',
|
|
'realview-switcheroo-full',
|
|
'realview64-simple-atomic',
|
|
'realview64-simple-atomic-checkpoint',
|
|
'realview64-simple-atomic-dual',
|
|
'realview64-simple-timing',
|
|
'realview64-simple-timing-dual',
|
|
'realview64-o3',
|
|
'realview64-o3-checker',
|
|
'realview64-o3-dual',
|
|
'realview64-minor',
|
|
'realview64-minor-dual',
|
|
'realview64-switcheroo-atomic',
|
|
'realview64-switcheroo-timing',
|
|
'realview64-switcheroo-o3',
|
|
'realview64-switcheroo-full',
|
|
),
|
|
|
|
("sparc", None) : (
|
|
't1000-simple-atomic',
|
|
't1000-simple-x86',
|
|
),
|
|
|
|
("timing", None) : (
|
|
'pc-simple-atomic',
|
|
'pc-simple-timing',
|
|
'pc-o3-timing',
|
|
'pc-switcheroo-full',
|
|
),
|
|
|
|
("x86", "hsail") : (
|
|
'gpu',
|
|
),
|
|
}
|
|
|
|
generic_configs = (
|
|
'simple-atomic',
|
|
'simple-atomic-mp',
|
|
'simple-timing',
|
|
'simple-timing-mp',
|
|
|
|
'minor-timing',
|
|
'minor-timing-mp',
|
|
|
|
'o3-timing',
|
|
'o3-timing-mt',
|
|
'o3-timing-mp',
|
|
|
|
'rubytest',
|
|
'memcheck',
|
|
'memtest',
|
|
'memtest-filter',
|
|
'tgen-simple-mem',
|
|
'tgen-dram-ctrl',
|
|
|
|
'learning-gem5-p1-simple',
|
|
'learning-gem5-p1-two-level',
|
|
)
|
|
|
|
all_categories = ("quick", "long")
|
|
all_modes = ("fs", "se")
|
|
|
|
class Test(object):
|
|
"""Test case base class.
|
|
|
|
Test cases consists of one or more test units that are run in two
|
|
phases. A run phase (units produced by run_units() and a verify
|
|
phase (units from verify_units()). The verify phase is skipped if
|
|
the run phase fails.
|
|
|
|
"""
|
|
|
|
__metaclass__ = ABCMeta
|
|
|
|
def __init__(self, name):
|
|
self.test_name = name
|
|
|
|
@abstractmethod
|
|
def ref_files(self):
|
|
"""Get a list of reference files used by this test case"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def run_units(self):
|
|
"""Units (typically RunGem5 instances) that describe the run phase of
|
|
this test.
|
|
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def verify_units(self):
|
|
"""Verify the output from the run phase (see run_units())."""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def update_ref(self):
|
|
"""Update reference files with files from a test run"""
|
|
pass
|
|
|
|
def run(self):
|
|
"""Run this test case and return a list of results"""
|
|
|
|
run_results = [ u.run() for u in self.run_units() ]
|
|
run_ok = all([not r.skipped() and r for r in run_results ])
|
|
|
|
verify_results = [
|
|
u.run() if run_ok else u.skip()
|
|
for u in self.verify_units()
|
|
]
|
|
|
|
return TestResult(self.test_name,
|
|
run_results=run_results,
|
|
verify_results=verify_results)
|
|
|
|
def __str__(self):
|
|
return self.test_name
|
|
|
|
class ClassicTest(Test):
|
|
# The diff ignore list contains all files that shouldn't be diffed
|
|
# using DiffOutFile. These files typically use special-purpose
|
|
# diff tools (e.g., DiffStatFile).
|
|
diff_ignore_files = FileIgnoreList(
|
|
names=(
|
|
# Stat files use a special stat differ
|
|
"stats.txt",
|
|
), rex=(
|
|
))
|
|
|
|
# These files should never be included in the list of
|
|
# reference files. This list should include temporary files
|
|
# and other files that we don't care about.
|
|
ref_ignore_files = FileIgnoreList(
|
|
names=(
|
|
), rex=(
|
|
# Mercurial sometimes leaves backups when applying MQ patches
|
|
r"\.orig$",
|
|
r"\.rej$",
|
|
))
|
|
|
|
def __init__(self, gem5, output_dir, config_tuple,
|
|
timeout=None,
|
|
skip=False, skip_diff_out=False, skip_diff_stat=False):
|
|
|
|
super(ClassicTest, self).__init__("/".join(config_tuple))
|
|
|
|
ct = config_tuple
|
|
|
|
self.gem5 = os.path.abspath(gem5)
|
|
self.script = os.path.join(_test_base, "run.py")
|
|
self.config_tuple = ct
|
|
self.timeout = timeout
|
|
|
|
self.output_dir = output_dir
|
|
self.ref_dir = os.path.join(_test_base,
|
|
ct.category, ct.mode, ct.workload,
|
|
"ref", ct.isa, ct.os, ct.config)
|
|
self.skip_run = skip
|
|
self.skip_diff_out = skip or skip_diff_out
|
|
self.skip_diff_stat = skip or skip_diff_stat
|
|
|
|
def ref_files(self):
|
|
ref_dir = os.path.abspath(self.ref_dir)
|
|
for root, dirs, files in os.walk(ref_dir, topdown=False):
|
|
for f in files:
|
|
fpath = os.path.join(root[len(ref_dir) + 1:], f)
|
|
if fpath not in ClassicTest.ref_ignore_files:
|
|
yield fpath
|
|
|
|
def run_units(self):
|
|
args = [
|
|
self.script,
|
|
"/".join(self.config_tuple),
|
|
]
|
|
|
|
return [
|
|
RunGem5(self.gem5, args,
|
|
ref_dir=self.ref_dir, test_dir=self.output_dir,
|
|
skip=self.skip_run),
|
|
]
|
|
|
|
def verify_units(self):
|
|
return [
|
|
DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir,
|
|
skip=self.skip_diff_stat)
|
|
] + [
|
|
DiffOutFile(f,
|
|
ref_dir=self.ref_dir, test_dir=self.output_dir,
|
|
skip=self.skip_diff_out)
|
|
for f in self.ref_files()
|
|
if f not in ClassicTest.diff_ignore_files
|
|
]
|
|
|
|
def update_ref(self):
|
|
for fname in self.ref_files():
|
|
shutil.copy(
|
|
os.path.join(self.output_dir, fname),
|
|
os.path.join(self.ref_dir, fname))
|
|
|
|
def parse_test_filter(test_filter):
|
|
wildcards = ("", "*")
|
|
|
|
_filter = list(test_filter.split("/"))
|
|
if len(_filter) > 3:
|
|
raise RuntimeError("Illegal test filter string")
|
|
_filter += [ "", ] * (3 - len(_filter))
|
|
|
|
isa, cat, mode = _filter
|
|
|
|
if isa in wildcards:
|
|
raise RuntimeError("No ISA specified")
|
|
|
|
cat = all_categories if cat in wildcards else (cat, )
|
|
mode = all_modes if mode in wildcards else (mode, )
|
|
|
|
return isa, cat, mode
|
|
|
|
def get_tests(isa,
|
|
categories=all_categories, modes=all_modes,
|
|
ruby_protocol=None, gpu_isa=None):
|
|
|
|
# Generate a list of candidate configs
|
|
configs = list(arch_configs.get((isa, gpu_isa), []))
|
|
|
|
if (isa, gpu_isa) == ("x86", "hsail"):
|
|
if ruby_protocol == "GPU_RfO":
|
|
configs += ['gpu-randomtest']
|
|
else:
|
|
configs += generic_configs
|
|
|
|
if ruby_protocol == 'MI_example':
|
|
configs += [ "%s-ruby" % (c, ) for c in configs ]
|
|
elif ruby_protocol is not None:
|
|
# Override generic ISA configs when using Ruby (excluding
|
|
# MI_example which is included in all ISAs by default). This
|
|
# reduces the number of generic tests we re-run for when
|
|
# compiling Ruby targets.
|
|
configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
|
|
|
|
# /(quick|long)/(fs|se)/workload/ref/arch/guest/config/
|
|
for conf_script in configs:
|
|
for cat in categories:
|
|
for mode in modes:
|
|
mode_dir = os.path.join(_test_base, cat, mode)
|
|
if not os.path.exists(mode_dir):
|
|
continue
|
|
|
|
for workload in os.listdir(mode_dir):
|
|
isa_dir = os.path.join(mode_dir, workload, "ref", isa)
|
|
if not os.path.isdir(isa_dir):
|
|
continue
|
|
|
|
for _os in os.listdir(isa_dir):
|
|
test_dir = os.path.join(isa_dir, _os, conf_script)
|
|
if not os.path.exists(test_dir) or \
|
|
os.path.exists(os.path.join(test_dir, "skip")):
|
|
continue
|
|
|
|
yield ClassicConfig(cat, mode, workload, isa, _os,
|
|
conf_script)
|