tests: Expand the reliability test suite.

Change-Id: Ic7f90f2d4edae1f72f98b34bda70891330c27941
This commit is contained in:
Cristiano Giuffrida 2014-12-22 03:38:50 +01:00 committed by David van Moolenbroek
parent 43065aa378
commit 0c474453d1
2 changed files with 286 additions and 85 deletions

View file

@ -58,19 +58,20 @@ service_get_policies(struct policies * pol, index_t slot)
/* iommu */
{ .label = "amddev", .policy_str = "" },
/* net */
{ .label = "3c90x", .policy_str = "restart" },
{ .label = "atl2", .policy_str = "restart" },
{ .label = "dec21140A", .policy_str = "restart" },
{ .label = "dp8390", .policy_str = "restart" },
{ .label = "dpeth", .policy_str = "restart" },
{ .label = "e1000", .policy_str = "restart" },
{ .label = "fxp", .policy_str = "restart" },
{ .label = "lance", .policy_str = "restart" },
{ .label = "lan8710a", .policy_str = "restart" },
{ .label = "rtl8139", .policy_str = "restart" },
{ .label = "rtl8169", .policy_str = "restart" },
{ .label = "3c90x", .policy_str = "reset" },
{ .label = "atl2", .policy_str = "reset" },
{ .label = "dec21140A", .policy_str = "reset" },
{ .label = "dp8390", .policy_str = "reset" },
{ .label = "dpeth", .policy_str = "reset" },
{ .label = "e1000", .policy_str = "reset" },
{ .label = "fxp", .policy_str = "reset" },
{ .label = "lance", .policy_str = "reset" },
{ .label = "lan8710a", .policy_str = "reset" },
{ .label = "orinoco", .policy_str = "reset" },
{ .label = "rtl8139", .policy_str = "reset" },
{ .label = "rtl8169", .policy_str = "reset" },
{ .label = "uds", .policy_str = "reset" },
{ .label = "virtio_net", .policy_str = "restart" },
{ .label = "virtio_net", .policy_str = "reset" },
/* power */
{ .label = "acpi", .policy_str = "" },
{ .label = "tps65217", .policy_str = "" },
@ -97,7 +98,7 @@ service_get_policies(struct policies * pol, index_t slot)
{ .label = "random", .policy_str = "restart" },
/* tty */
{ .label = "pty", .policy_str = "restart" },
{ .label = "tty", .policy_str = "" },
{ .label = "tty", .policy_str = "restart" },
/* usb */
{ .label = "usbd", .policy_str = "" },
{ .label = "usb_hub", .policy_str = "" },
@ -111,24 +112,24 @@ service_get_policies(struct policies * pol, index_t slot)
{ .label = "ext2", .policy_str = "" },
{ .label = "hgfs", .policy_str = "" },
{ .label = "isofs", .policy_str = "" },
{ .label = "mfs", .policy_str = "" },
{ .label = "pfs", .policy_str = "" },
{ .label = "procfs", .policy_str = "" },
{ .label = "mfs", .policy_str = "restart" },
{ .label = "pfs", .policy_str = "restart" },
{ .label = "procfs", .policy_str = "restart" },
{ .label = "ptyfs", .policy_str = "" },
{ .label = "vbfs", .policy_str = "" },
/* net */
{ .label = "inet", .policy_str = "reset" },
{ .label = "lwip", .policy_str = "" },
/* servers */
{ .label = "devman", .policy_str = "" },
{ .label = "ds", .policy_str = "" },
{ .label = "devman", .policy_str = "restart" },
{ .label = "ds", .policy_str = "restart" },
{ .label = "input", .policy_str = "reset" },
{ .label = "ipc", .policy_str = "restart" },
{ .label = "is", .policy_str = "restart" },
{ .label = "pm", .policy_str = "" },
{ .label = "rs", .policy_str = "" },
{ .label = "sched", .policy_str = "" },
{ .label = "vfs", .policy_str = "" },
{ .label = "pm", .policy_str = "restart" },
{ .label = "rs", .policy_str = "restart" },
{ .label = "sched", .policy_str = "restart" },
{ .label = "vfs", .policy_str = "restart" },
{ .label = "vm", .policy_str = "" },
//{ .label = "", .policy_str = "" },
};
@ -154,6 +155,30 @@ service_get_policies(struct policies * pol, index_t slot)
return pol[slot].formatted;
}
/* Returns a ASCIIZ string encoding RS flags. */
static const char *
service_get_flags(index_t slot)
{
static char str[10];
int flags, sys_flags;
flags = rproc.proc[slot].r_flags;
sys_flags = rproc.pub[slot].sys_flags;
str[0] = (flags & RS_ACTIVE) ? 'A' : '-';
str[1] = (flags & RS_UPDATING) ? 'U' : '-';
str[2] = (flags & RS_EXITING) ? 'E' : '-';
str[3] = (flags & RS_NOPINGREPLY) ? 'N' : '-';
str[4] = (sys_flags & SF_USE_COPY) ? 'C' : '-';
str[5] = (sys_flags & SF_USE_REPL) ? 'R' : '-';
str[6] = (sys_flags & SF_NEED_COPY) ? 'c' : '-';
str[7] = (sys_flags & SF_NEED_REPL) ? 'r' : '-';
str[8] = (sys_flags & SF_CORE_SRV) ? 's' : '-';
str[9] = '\0';
return str;
}
/*
* Return whether a slot is in use and active. The purpose of this check is
* to ensure that after eliminating all slots that do not pass this check, we
@ -310,5 +335,6 @@ service_read(struct inode * node)
buf_printf("endpoint: %d\n", rpub->endpoint);
buf_printf("pid: %d\n", rp->r_pid);
buf_printf("restarts: %d\n", rp->r_restarts);
buf_printf("flags: %s\n", service_get_flags(slot));
buf_printf("policies: %s\n", service_get_policies(policies, slot));
}

View file

@ -5,7 +5,8 @@
# for each restatability policy call the policy test function if it is
# supported. No accounting of failed / successful test is done, as a
# failed test can currently provoque cascading effects, so instead we
# fail the test as a whole on the first failurei found.
# fail the test as a whole on the first failure is found. Live update tests
# are currently policy-agnostic.
#
# Supported policies have to be in the POLICIES variable, and define a test
# function.
@ -13,6 +14,7 @@
# Known limitations:
# - Currently not all recovery policies are tested
# - Running this test under X11 hangs the X server
# - Live update tests do not test rollback situations
#
# To add a new policy, you have to do the following:
# 1. Add the policy into the active policies array by:
@ -28,8 +30,7 @@
# Currently known policies:
# /* user | endpoint */
# POL_RESET, /* visible | change */
# POL_RESTART, /* transparent | preserved */
# POL_LIVE_UPDATE /* transparent | preserved */
# POL_RESTART /* transparent | preserved */
#######################################################################
# Utility functions & global state initializations
@ -59,7 +60,7 @@ wait_for_service() {
do
sleep 1
retry=$((${retry} + 1))
test -f $1 && break
test -f $1 && test $(get_value restarts $1) -ne $2 && break
done
}
@ -78,8 +79,8 @@ pol_restart() {
restarts_pre=$(get_value restarts ${service})
endpoint_pre=$(get_value endpoint ${service})
service refresh ${label}
wait_for_service ${service}
service fi ${label}
wait_for_service ${service} ${restarts_pre}
restarts_post=$(get_value restarts ${service})
endpoint_post=$(get_value endpoint ${service})
@ -100,21 +101,25 @@ POLICIES="${POLICIES} reset"
pol_reset() {
local label service
local endpoint_pre endpoint_post
local restarts_pre restarts_post
service=$1
label=$2
restarts_pre=$(get_value restarts ${service})
endpoint_pre=$(get_value endpoint ${service})
service refresh ${label}
wait_for_service ${service}
service fi ${label}
wait_for_service ${service} ${restarts_pre}
restarts_post=$(get_value restarts ${service})
endpoint_post=$(get_value endpoint ${service})
# This policy doesn't guarantee the endpoint to be kept, but there
# is a slight chance that it will actualy stay the same, and fail
# the test.
if [ ! ${endpoint_post} -eq ${endpoint_pre} ]
if [ ${restarts_post} -gt ${restarts_pre} \
-a ${endpoint_post} -ne ${endpoint_pre} ]
then
echo ok
else
@ -122,6 +127,138 @@ pol_reset() {
fi
}
#######################################################################
# Live update test
#######################################################################
lu_test_one() {
local label=$1
local prog=$2
local result=$3
lu_opts=${lu_opts:-}
lu_maxtime=${lu_maxtime:-3HZ}
lu_state=${lu_state:-1}
service ${lu_opts} update ${prog} -label ${label} -maxtime ${lu_maxtime} -state ${lu_state}
if [ $? -ne $result ]
then
echo not ok
return 1
else
return 0
fi
}
lu_test() {
local label service
local endpoint_pre endpoint_post
local restarts_pre restarts_post
service=$1
label=$2
restarts_pre=$(get_value restarts ${service})
endpoint_pre=$(get_value endpoint ${service})
lu_test_one ${label} self 0 || return
if ! echo "vm pm vfs rs" | grep -q ${label}
then
lu_opts="-o" lu_test_one ${label} self 0 || return
fi
lu_opts="-x" lu_test_one ${label} self 200 || return
if ! echo "rs" | grep -q ${label}
then
lu_opts="-y" lu_test_one ${label} self 78 || return
lu_maxtime="1HZ" lu_opts="-z" lu_test_one ${label} self 4 || return
fi
lu_maxtime="1HZ" lu_state="5" lu_test_one ${label} self 4 || return
restarts_post=$(get_value restarts ${service})
endpoint_post=$(get_value endpoint ${service})
# Make sure endpoint and restarts are preserved
if [ ${restarts_post} -eq ${restarts_pre} \
-a ${endpoint_post} -eq ${endpoint_pre} ]
then
echo ok
else
echo not ok
fi
}
multi_lu_test_one() {
local result=$1
shift
local labels="$*"
local ret=0
local index=0
local once_index=1
lu_opts=${lu_opts:-}
lu_maxtime=${lu_maxtime:-3HZ}
lu_state=${lu_state:-1}
lu_opts_once=${lu_opts_once:-$lu_opts}
lu_maxtime_once=${lu_maxtime_once:-$lu_maxtime}
lu_state_once=${lu_state_once:-$lu_state}
for label in ${labels}
do
if [ $index -eq $once_index ]
then
service ${lu_opts_once} -q update self -label ${label} -maxtime ${lu_maxtime_once} -state ${lu_state_once} || ret=1
else
service ${lu_opts} -q update self -label ${label} -maxtime ${lu_maxtime} -state ${lu_state} || ret=1
fi
index=`expr $index + 1`
done
service sysctl upd_run
if [ $? -ne $result ]
then
ret=1
fi
if [ $ret -eq 1 ]
then
echo not ok
fi
return $ret
}
multi_lu_test() {
local labels="$*"
multi_lu_test_one 0 ${labels} || return
lu_opts_once="-x" multi_lu_test_one 200 ${labels} || return
lu_opts_once="-y" multi_lu_test_one 200 ${labels} || return
lu_maxtime_once="1HZ" lu_opts_once="-z" multi_lu_test_one 200 ${labels} || return
lu_maxtime_once="1HZ" lu_state_once="5" multi_lu_test_one 4 ${labels} || return
echo ok
}
#######################################################################
# Service management routines
#######################################################################
prepare_service() {
local label service
service=$1
label=$2
flags=$(get_value flags ${service})
echo $flags | grep -q 'r' || return 0
echo $flags | grep -q 'R' && return 0
service clone $label
return 1
}
cleanup_service() {
local label
label=$1
service unclone $label
}
#######################################################################
# main()
#######################################################################
@ -136,27 +273,65 @@ main() {
exit 1
fi
labels=$(echo /proc/service/*)
for label in ${labels}
services=$(echo /proc/service/*)
for service in ${services}
do
service_policies=$(grep policies ${label}|cut -d: -f2)
label=$(basename ${service})
service_policies=$(grep policies ${service}|cut -d: -f2)
for pol in ${service_policies}
do
# Check if the supported policy is under test
if echo "${POLICIES}" | grep -q ${pol}
then
echo "# testing ${label} :: ${pol}"
result=$(pol_${pol} ${label} $(basename ${label}))
#pol_${pol} ${label} $(basename ${label})
#result="FAILED"
cleanup=0
prepare_service ${service} ${label} || cleanup=1
result=$(pol_${pol} ${service} ${label})
if [ "x${result}" != "xok" ]
then
echo "not ok # failed ${label}, ${pol}"
exit 1
fi
if [ $cleanup -eq 1 ]
then
cleanup_service ${label}
fi
fi
done
done
multi_lu_labels=""
for service in ${services}
do
label=$(basename ${service})
service_policies=$(grep policies ${service}|cut -d: -f2)
if echo "${service_policies}" | grep -q "[a-zA-Z]"
then
echo "# testing ${label} :: live update+rollback"
result=$(lu_test ${service} ${label})
if [ "x${result}" != "xok" ]
then
echo "not ok # failed ${label}, live update+rollback"
exit 1
fi
if [ "x${label}" = "xrs" ]
then
continue
fi
service_flags=$(get_value flags ${service})
if echo $service_flags | grep -q 's'
then
multi_lu_labels="${multi_lu_labels} ${label}"
fi
fi
done
multi_lu_labels="${multi_lu_labels} rs"
echo "# testing ${multi_lu_labels} :: whole-OS live update+rollback"
result=$(multi_lu_test $multi_lu_labels)
if [ "x${result}" != "xok" ]
then
echo "not ok # failed whole-OS live update+rollback"
exit 1
fi
echo ok
exit 0