tests: Expand the reliability test suite.
Change-Id: Ic7f90f2d4edae1f72f98b34bda70891330c27941
This commit is contained in:
Cristiano Giuffrida
David van Moolenbroek
parent
43065aa378
commit
0c474453d1
2 changed files with 286 additions and 85 deletions
|
|
@ -58,19 +58,20 @@ service_get_policies(struct policies * pol, index_t slot)
|
|||
/* iommu */
|
||||
{ .label = "amddev", .policy_str = "" },
|
||||
/* net */
|
||||
{ .label = "3c90x", .policy_str = "restart" },
|
||||
{ .label = "atl2", .policy_str = "restart" },
|
||||
{ .label = "dec21140A", .policy_str = "restart" },
|
||||
{ .label = "dp8390", .policy_str = "restart" },
|
||||
{ .label = "dpeth", .policy_str = "restart" },
|
||||
{ .label = "e1000", .policy_str = "restart" },
|
||||
{ .label = "fxp", .policy_str = "restart" },
|
||||
{ .label = "lance", .policy_str = "restart" },
|
||||
{ .label = "lan8710a", .policy_str = "restart" },
|
||||
{ .label = "rtl8139", .policy_str = "restart" },
|
||||
{ .label = "rtl8169", .policy_str = "restart" },
|
||||
{ .label = "3c90x", .policy_str = "reset" },
|
||||
{ .label = "atl2", .policy_str = "reset" },
|
||||
{ .label = "dec21140A", .policy_str = "reset" },
|
||||
{ .label = "dp8390", .policy_str = "reset" },
|
||||
{ .label = "dpeth", .policy_str = "reset" },
|
||||
{ .label = "e1000", .policy_str = "reset" },
|
||||
{ .label = "fxp", .policy_str = "reset" },
|
||||
{ .label = "lance", .policy_str = "reset" },
|
||||
{ .label = "lan8710a", .policy_str = "reset" },
|
||||
{ .label = "orinoco", .policy_str = "reset" },
|
||||
{ .label = "rtl8139", .policy_str = "reset" },
|
||||
{ .label = "rtl8169", .policy_str = "reset" },
|
||||
{ .label = "uds", .policy_str = "reset" },
|
||||
{ .label = "virtio_net", .policy_str = "restart" },
|
||||
{ .label = "virtio_net", .policy_str = "reset" },
|
||||
/* power */
|
||||
{ .label = "acpi", .policy_str = "" },
|
||||
{ .label = "tps65217", .policy_str = "" },
|
||||
|
|
@ -97,7 +98,7 @@ service_get_policies(struct policies * pol, index_t slot)
|
|||
{ .label = "random", .policy_str = "restart" },
|
||||
/* tty */
|
||||
{ .label = "pty", .policy_str = "restart" },
|
||||
{ .label = "tty", .policy_str = "" },
|
||||
{ .label = "tty", .policy_str = "restart" },
|
||||
/* usb */
|
||||
{ .label = "usbd", .policy_str = "" },
|
||||
{ .label = "usb_hub", .policy_str = "" },
|
||||
|
|
@ -111,24 +112,24 @@ service_get_policies(struct policies * pol, index_t slot)
|
|||
{ .label = "ext2", .policy_str = "" },
|
||||
{ .label = "hgfs", .policy_str = "" },
|
||||
{ .label = "isofs", .policy_str = "" },
|
||||
{ .label = "mfs", .policy_str = "" },
|
||||
{ .label = "pfs", .policy_str = "" },
|
||||
{ .label = "procfs", .policy_str = "" },
|
||||
{ .label = "mfs", .policy_str = "restart" },
|
||||
{ .label = "pfs", .policy_str = "restart" },
|
||||
{ .label = "procfs", .policy_str = "restart" },
|
||||
{ .label = "ptyfs", .policy_str = "" },
|
||||
{ .label = "vbfs", .policy_str = "" },
|
||||
/* net */
|
||||
{ .label = "inet", .policy_str = "reset" },
|
||||
{ .label = "lwip", .policy_str = "" },
|
||||
/* servers */
|
||||
{ .label = "devman", .policy_str = "" },
|
||||
{ .label = "ds", .policy_str = "" },
|
||||
{ .label = "devman", .policy_str = "restart" },
|
||||
{ .label = "ds", .policy_str = "restart" },
|
||||
{ .label = "input", .policy_str = "reset" },
|
||||
{ .label = "ipc", .policy_str = "restart" },
|
||||
{ .label = "is", .policy_str = "restart" },
|
||||
{ .label = "pm", .policy_str = "" },
|
||||
{ .label = "rs", .policy_str = "" },
|
||||
{ .label = "sched", .policy_str = "" },
|
||||
{ .label = "vfs", .policy_str = "" },
|
||||
{ .label = "pm", .policy_str = "restart" },
|
||||
{ .label = "rs", .policy_str = "restart" },
|
||||
{ .label = "sched", .policy_str = "restart" },
|
||||
{ .label = "vfs", .policy_str = "restart" },
|
||||
{ .label = "vm", .policy_str = "" },
|
||||
//{ .label = "", .policy_str = "" },
|
||||
};
|
||||
|
|
@ -154,6 +155,30 @@ service_get_policies(struct policies * pol, index_t slot)
|
|||
return pol[slot].formatted;
|
||||
}
|
||||
|
||||
/* Returns a ASCIIZ string encoding RS flags. */
|
||||
static const char *
|
||||
service_get_flags(index_t slot)
|
||||
{
|
||||
static char str[10];
|
||||
int flags, sys_flags;
|
||||
|
||||
flags = rproc.proc[slot].r_flags;
|
||||
sys_flags = rproc.pub[slot].sys_flags;
|
||||
|
||||
str[0] = (flags & RS_ACTIVE) ? 'A' : '-';
|
||||
str[1] = (flags & RS_UPDATING) ? 'U' : '-';
|
||||
str[2] = (flags & RS_EXITING) ? 'E' : '-';
|
||||
str[3] = (flags & RS_NOPINGREPLY) ? 'N' : '-';
|
||||
str[4] = (sys_flags & SF_USE_COPY) ? 'C' : '-';
|
||||
str[5] = (sys_flags & SF_USE_REPL) ? 'R' : '-';
|
||||
str[6] = (sys_flags & SF_NEED_COPY) ? 'c' : '-';
|
||||
str[7] = (sys_flags & SF_NEED_REPL) ? 'r' : '-';
|
||||
str[8] = (sys_flags & SF_CORE_SRV) ? 's' : '-';
|
||||
str[9] = '\0';
|
||||
|
||||
return str;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return whether a slot is in use and active. The purpose of this check is
|
||||
* to ensure that after eliminating all slots that do not pass this check, we
|
||||
|
|
@ -310,5 +335,6 @@ service_read(struct inode * node)
|
|||
buf_printf("endpoint: %d\n", rpub->endpoint);
|
||||
buf_printf("pid: %d\n", rp->r_pid);
|
||||
buf_printf("restarts: %d\n", rp->r_restarts);
|
||||
buf_printf("flags: %s\n", service_get_flags(slot));
|
||||
buf_printf("policies: %s\n", service_get_policies(policies, slot));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@
|
|||
# for each restatability policy call the policy test function if it is
|
||||
# supported. No accounting of failed / successful test is done, as a
|
||||
# failed test can currently provoque cascading effects, so instead we
|
||||
# fail the test as a whole on the first failurei found.
|
||||
# fail the test as a whole on the first failure is found. Live update tests
|
||||
# are currently policy-agnostic.
|
||||
#
|
||||
# Supported policies have to be in the POLICIES variable, and define a test
|
||||
# function.
|
||||
|
|
@ -13,6 +14,7 @@
|
|||
# Known limitations:
|
||||
# - Currently not all recovery policies are tested
|
||||
# - Running this test under X11 hangs the X server
|
||||
# - Live update tests do not test rollback situations
|
||||
#
|
||||
# To add a new policy, you have to do the following:
|
||||
# 1. Add the policy into the active policies array by:
|
||||
|
|
@ -28,8 +30,7 @@
|
|||
# Currently known policies:
|
||||
# /* user | endpoint */
|
||||
# POL_RESET, /* visible | change */
|
||||
# POL_RESTART, /* transparent | preserved */
|
||||
# POL_LIVE_UPDATE /* transparent | preserved */
|
||||
# POL_RESTART /* transparent | preserved */
|
||||
|
||||
#######################################################################
|
||||
# Utility functions & global state initializations
|
||||
|
|
@ -59,7 +60,7 @@ wait_for_service() {
|
|||
do
|
||||
sleep 1
|
||||
retry=$((${retry} + 1))
|
||||
test -f $1 && break
|
||||
test -f $1 && test $(get_value restarts $1) -ne $2 && break
|
||||
done
|
||||
}
|
||||
|
||||
|
|
@ -78,8 +79,8 @@ pol_restart() {
|
|||
restarts_pre=$(get_value restarts ${service})
|
||||
endpoint_pre=$(get_value endpoint ${service})
|
||||
|
||||
service refresh ${label}
|
||||
wait_for_service ${service}
|
||||
service fi ${label}
|
||||
wait_for_service ${service} ${restarts_pre}
|
||||
|
||||
restarts_post=$(get_value restarts ${service})
|
||||
endpoint_post=$(get_value endpoint ${service})
|
||||
|
|
@ -100,21 +101,25 @@ POLICIES="${POLICIES} reset"
|
|||
pol_reset() {
|
||||
local label service
|
||||
local endpoint_pre endpoint_post
|
||||
local restarts_pre restarts_post
|
||||
|
||||
service=$1
|
||||
label=$2
|
||||
|
||||
restarts_pre=$(get_value restarts ${service})
|
||||
endpoint_pre=$(get_value endpoint ${service})
|
||||
|
||||
service refresh ${label}
|
||||
wait_for_service ${service}
|
||||
service fi ${label}
|
||||
wait_for_service ${service} ${restarts_pre}
|
||||
|
||||
restarts_post=$(get_value restarts ${service})
|
||||
endpoint_post=$(get_value endpoint ${service})
|
||||
|
||||
# This policy doesn't guarantee the endpoint to be kept, but there
|
||||
# is a slight chance that it will actualy stay the same, and fail
|
||||
# the test.
|
||||
if [ ! ${endpoint_post} -eq ${endpoint_pre} ]
|
||||
if [ ${restarts_post} -gt ${restarts_pre} \
|
||||
-a ${endpoint_post} -ne ${endpoint_pre} ]
|
||||
then
|
||||
echo ok
|
||||
else
|
||||
|
|
@ -122,6 +127,138 @@ pol_reset() {
|
|||
fi
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# Live update test
|
||||
#######################################################################
|
||||
lu_test_one() {
|
||||
local label=$1
|
||||
local prog=$2
|
||||
local result=$3
|
||||
lu_opts=${lu_opts:-}
|
||||
lu_maxtime=${lu_maxtime:-3HZ}
|
||||
lu_state=${lu_state:-1}
|
||||
|
||||
service ${lu_opts} update ${prog} -label ${label} -maxtime ${lu_maxtime} -state ${lu_state}
|
||||
if [ $? -ne $result ]
|
||||
then
|
||||
echo not ok
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
lu_test() {
|
||||
local label service
|
||||
local endpoint_pre endpoint_post
|
||||
local restarts_pre restarts_post
|
||||
|
||||
service=$1
|
||||
label=$2
|
||||
|
||||
restarts_pre=$(get_value restarts ${service})
|
||||
endpoint_pre=$(get_value endpoint ${service})
|
||||
|
||||
lu_test_one ${label} self 0 || return
|
||||
if ! echo "vm pm vfs rs" | grep -q ${label}
|
||||
then
|
||||
lu_opts="-o" lu_test_one ${label} self 0 || return
|
||||
fi
|
||||
lu_opts="-x" lu_test_one ${label} self 200 || return
|
||||
if ! echo "rs" | grep -q ${label}
|
||||
then
|
||||
lu_opts="-y" lu_test_one ${label} self 78 || return
|
||||
lu_maxtime="1HZ" lu_opts="-z" lu_test_one ${label} self 4 || return
|
||||
fi
|
||||
lu_maxtime="1HZ" lu_state="5" lu_test_one ${label} self 4 || return
|
||||
|
||||
restarts_post=$(get_value restarts ${service})
|
||||
endpoint_post=$(get_value endpoint ${service})
|
||||
|
||||
# Make sure endpoint and restarts are preserved
|
||||
if [ ${restarts_post} -eq ${restarts_pre} \
|
||||
-a ${endpoint_post} -eq ${endpoint_pre} ]
|
||||
then
|
||||
echo ok
|
||||
else
|
||||
echo not ok
|
||||
fi
|
||||
}
|
||||
|
||||
multi_lu_test_one() {
|
||||
local result=$1
|
||||
shift
|
||||
local labels="$*"
|
||||
local ret=0
|
||||
local index=0
|
||||
local once_index=1
|
||||
|
||||
lu_opts=${lu_opts:-}
|
||||
lu_maxtime=${lu_maxtime:-3HZ}
|
||||
lu_state=${lu_state:-1}
|
||||
lu_opts_once=${lu_opts_once:-$lu_opts}
|
||||
lu_maxtime_once=${lu_maxtime_once:-$lu_maxtime}
|
||||
lu_state_once=${lu_state_once:-$lu_state}
|
||||
|
||||
for label in ${labels}
|
||||
do
|
||||
if [ $index -eq $once_index ]
|
||||
then
|
||||
service ${lu_opts_once} -q update self -label ${label} -maxtime ${lu_maxtime_once} -state ${lu_state_once} || ret=1
|
||||
else
|
||||
service ${lu_opts} -q update self -label ${label} -maxtime ${lu_maxtime} -state ${lu_state} || ret=1
|
||||
fi
|
||||
index=`expr $index + 1`
|
||||
done
|
||||
service sysctl upd_run
|
||||
if [ $? -ne $result ]
|
||||
then
|
||||
ret=1
|
||||
fi
|
||||
if [ $ret -eq 1 ]
|
||||
then
|
||||
echo not ok
|
||||
fi
|
||||
return $ret
|
||||
}
|
||||
|
||||
multi_lu_test() {
|
||||
local labels="$*"
|
||||
|
||||
multi_lu_test_one 0 ${labels} || return
|
||||
lu_opts_once="-x" multi_lu_test_one 200 ${labels} || return
|
||||
lu_opts_once="-y" multi_lu_test_one 200 ${labels} || return
|
||||
lu_maxtime_once="1HZ" lu_opts_once="-z" multi_lu_test_one 200 ${labels} || return
|
||||
lu_maxtime_once="1HZ" lu_state_once="5" multi_lu_test_one 4 ${labels} || return
|
||||
|
||||
echo ok
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# Service management routines
|
||||
#######################################################################
|
||||
prepare_service() {
|
||||
local label service
|
||||
|
||||
service=$1
|
||||
label=$2
|
||||
|
||||
flags=$(get_value flags ${service})
|
||||
echo $flags | grep -q 'r' || return 0
|
||||
echo $flags | grep -q 'R' && return 0
|
||||
|
||||
service clone $label
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup_service() {
|
||||
local label
|
||||
|
||||
label=$1
|
||||
|
||||
service unclone $label
|
||||
}
|
||||
|
||||
#######################################################################
|
||||
# main()
|
||||
#######################################################################
|
||||
|
|
@ -136,27 +273,65 @@ main() {
|
|||
exit 1
|
||||
fi
|
||||
|
||||
labels=$(echo /proc/service/*)
|
||||
for label in ${labels}
|
||||
services=$(echo /proc/service/*)
|
||||
for service in ${services}
|
||||
do
|
||||
service_policies=$(grep policies ${label}|cut -d: -f2)
|
||||
label=$(basename ${service})
|
||||
service_policies=$(grep policies ${service}|cut -d: -f2)
|
||||
for pol in ${service_policies}
|
||||
do
|
||||
# Check if the supported policy is under test
|
||||
if echo "${POLICIES}" | grep -q ${pol}
|
||||
then
|
||||
echo "# testing ${label} :: ${pol}"
|
||||
result=$(pol_${pol} ${label} $(basename ${label}))
|
||||
#pol_${pol} ${label} $(basename ${label})
|
||||
#result="FAILED"
|
||||
cleanup=0
|
||||
prepare_service ${service} ${label} || cleanup=1
|
||||
result=$(pol_${pol} ${service} ${label})
|
||||
if [ "x${result}" != "xok" ]
|
||||
then
|
||||
echo "not ok # failed ${label}, ${pol}"
|
||||
exit 1
|
||||
fi
|
||||
if [ $cleanup -eq 1 ]
|
||||
then
|
||||
cleanup_service ${label}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
multi_lu_labels=""
|
||||
for service in ${services}
|
||||
do
|
||||
label=$(basename ${service})
|
||||
service_policies=$(grep policies ${service}|cut -d: -f2)
|
||||
if echo "${service_policies}" | grep -q "[a-zA-Z]"
|
||||
then
|
||||
echo "# testing ${label} :: live update+rollback"
|
||||
result=$(lu_test ${service} ${label})
|
||||
if [ "x${result}" != "xok" ]
|
||||
then
|
||||
echo "not ok # failed ${label}, live update+rollback"
|
||||
exit 1
|
||||
fi
|
||||
if [ "x${label}" = "xrs" ]
|
||||
then
|
||||
continue
|
||||
fi
|
||||
service_flags=$(get_value flags ${service})
|
||||
if echo $service_flags | grep -q 's'
|
||||
then
|
||||
multi_lu_labels="${multi_lu_labels} ${label}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
multi_lu_labels="${multi_lu_labels} rs"
|
||||
echo "# testing ${multi_lu_labels} :: whole-OS live update+rollback"
|
||||
result=$(multi_lu_test $multi_lu_labels)
|
||||
if [ "x${result}" != "xok" ]
|
||||
then
|
||||
echo "not ok # failed whole-OS live update+rollback"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ok
|
||||
exit 0
|
||||
|
|
|
|||
Loading…
Reference in a new issue