tests: Expand the reliability test suite.

Change-Id: Ic7f90f2d4edae1f72f98b34bda70891330c27941

minix/fs/procfs/service.c

@@ -58,19 +58,20 @@ service_get_policies(struct policies * pol, index_t slot)
 		/* iommu */
 		{ .label = "amddev", .policy_str = "" },
 		/* net */
-		{ .label = "3c90x", .policy_str = "restart" },
+		{ .label = "3c90x", .policy_str = "reset" },
-                { .label = "atl2", .policy_str = "restart" },
+		{ .label = "atl2", .policy_str = "reset" },
-                { .label = "dec21140A", .policy_str = "restart" },
+		{ .label = "dec21140A", .policy_str = "reset" },
-                { .label = "dp8390", .policy_str = "restart" },
+		{ .label = "dp8390", .policy_str = "reset" },
-                { .label = "dpeth", .policy_str = "restart" },
+		{ .label = "dpeth", .policy_str = "reset" },
-                { .label = "e1000", .policy_str = "restart" },
+		{ .label = "e1000", .policy_str = "reset" },
-                { .label = "fxp", .policy_str = "restart" },
+		{ .label = "fxp", .policy_str = "reset" },
-                { .label = "lance", .policy_str = "restart" },
+		{ .label = "lance", .policy_str = "reset" },
-                { .label = "lan8710a", .policy_str = "restart" },
+		{ .label = "lan8710a", .policy_str = "reset" },
-                { .label = "rtl8139", .policy_str = "restart" },
+		{ .label = "orinoco", .policy_str = "reset" },
-                { .label = "rtl8169", .policy_str = "restart" },
+		{ .label = "rtl8139", .policy_str = "reset" },
+		{ .label = "rtl8169", .policy_str = "reset" },
 		{ .label = "uds", .policy_str = "reset" },
-                { .label = "virtio_net", .policy_str = "restart" },
+		{ .label = "virtio_net", .policy_str = "reset" },
 		/* power */
 		{ .label = "acpi", .policy_str = "" },
 		{ .label = "tps65217", .policy_str = "" },
@@ -97,7 +98,7 @@ service_get_policies(struct policies * pol, index_t slot)
 		{ .label = "random", .policy_str = "restart" },
 		/* tty */
 		{ .label = "pty", .policy_str = "restart" },
-                { .label = "tty", .policy_str = "" },
+		{ .label = "tty", .policy_str = "restart" },
 		/* usb */
 		{ .label = "usbd", .policy_str = "" },
 		{ .label = "usb_hub", .policy_str = "" },
@@ -111,24 +112,24 @@ service_get_policies(struct policies * pol, index_t slot)
 		{ .label = "ext2", .policy_str = "" },
 		{ .label = "hgfs", .policy_str = "" },
 		{ .label = "isofs", .policy_str = "" },
-                { .label = "mfs", .policy_str = "" },
+		{ .label = "mfs", .policy_str = "restart" },
-                { .label = "pfs", .policy_str = "" },
+		{ .label = "pfs", .policy_str = "restart" },
-                { .label = "procfs", .policy_str = "" },
+		{ .label = "procfs", .policy_str = "restart" },
 		{ .label = "ptyfs", .policy_str = "" },
 		{ .label = "vbfs", .policy_str = "" },
 		/* net */
 		{ .label = "inet", .policy_str = "reset" },
 		{ .label = "lwip", .policy_str = "" },
 		/* servers */
-                { .label = "devman", .policy_str = "" },
+		{ .label = "devman", .policy_str = "restart" },
-                { .label = "ds", .policy_str = "" },
+		{ .label = "ds", .policy_str = "restart" },
 		{ .label = "input", .policy_str = "reset" },
 		{ .label = "ipc", .policy_str = "restart" },
 		{ .label = "is", .policy_str = "restart" },
-                { .label = "pm", .policy_str = "" },
+		{ .label = "pm", .policy_str = "restart" },
-                { .label = "rs", .policy_str = "" },
+		{ .label = "rs", .policy_str = "restart" },
-                { .label = "sched", .policy_str = "" },
+		{ .label = "sched", .policy_str = "restart" },
-                { .label = "vfs", .policy_str = "" },
+		{ .label = "vfs", .policy_str = "restart" },
 		{ .label = "vm", .policy_str = "" },
 		//{ .label = "", .policy_str = "" },
 	};
@@ -154,6 +155,30 @@ service_get_policies(struct policies * pol, index_t slot)
 	return pol[slot].formatted;
 }
 
+/* Returns a ASCIIZ string encoding RS flags.  */
+static const char *
+service_get_flags(index_t slot)
+{
+	static char str[10];
+	int flags, sys_flags;
+
+	flags = rproc.proc[slot].r_flags;
+	sys_flags = rproc.pub[slot].sys_flags;
+
+	str[0] = (flags & RS_ACTIVE)        ? 'A' : '-';
+	str[1] = (flags & RS_UPDATING)      ? 'U' : '-';
+	str[2] = (flags & RS_EXITING)       ? 'E' : '-';
+	str[3] = (flags & RS_NOPINGREPLY)   ? 'N' : '-';
+	str[4] = (sys_flags & SF_USE_COPY)  ? 'C' : '-';
+	str[5] = (sys_flags & SF_USE_REPL)  ? 'R' : '-';
+	str[6] = (sys_flags & SF_NEED_COPY) ? 'c' : '-';
+	str[7] = (sys_flags & SF_NEED_REPL) ? 'r' : '-';
+	str[8] = (sys_flags & SF_CORE_SRV)  ? 's' : '-';
+	str[9] = '\0';
+
+	return str;
+}
+
 /*
  * Return whether a slot is in use and active.  The purpose of this check is
  * to ensure that after eliminating all slots that do not pass this check, we
@@ -310,5 +335,6 @@ service_read(struct inode * node)
 	buf_printf("endpoint: %d\n", rpub->endpoint);
 	buf_printf("pid:      %d\n", rp->r_pid);
 	buf_printf("restarts: %d\n", rp->r_restarts);
+	buf_printf("flags:    %s\n", service_get_flags(slot));
 	buf_printf("policies: %s\n", service_get_policies(policies, slot));
 }

minix/tests/testrelpol.sh

@@ -5,7 +5,8 @@
 #	for each restatability policy call the policy test function if it is
 #	supported. No accounting of failed / successful test is done, as a
 #	failed test can currently provoque cascading effects, so instead we
-#	fail the test as a whole on the first failurei found.
+#	fail the test as a whole on the first failure is found. Live update tests
+#	are currently policy-agnostic.
 #
 # Supported policies have to be in the POLICIES variable, and define a test
 # function.
@@ -13,6 +14,7 @@
 # Known limitations:
 #	 - Currently not all recovery policies are tested
 #	 - Running this test under X11 hangs the X server
+#	 - Live update tests do not test rollback situations
 #
 # To add a new policy, you have to do the following:
 #	1. Add the policy into the active policies array by:
@@ -28,8 +30,7 @@
 # Currently known policies:
 # 			/*	user	| endpoint	*/
 #	POL_RESET,	/* visible	|  change	*/
-#	POL_RESTART,	/* transparent	| preserved	*/
+#	POL_RESTART	/* transparent	| preserved	*/
-#	POL_LIVE_UPDATE	/* transparent	| preserved	*/
 
 #######################################################################
 # Utility functions & global state initializations
@@ -59,7 +60,7 @@ wait_for_service() {
 	do
 		sleep 1
 		retry=$((${retry} + 1))
-		test -f $1 && break
+		test -f $1 && test $(get_value restarts $1) -ne $2 && break
 	done
 }
 
@@ -78,8 +79,8 @@ pol_restart() {
 	restarts_pre=$(get_value restarts ${service})
 	endpoint_pre=$(get_value endpoint ${service})
 
-	service refresh ${label}
+	service fi ${label}
-	wait_for_service ${service}
+	wait_for_service ${service} ${restarts_pre}
 
 	restarts_post=$(get_value restarts ${service})
 	endpoint_post=$(get_value endpoint ${service})
@@ -100,21 +101,25 @@ POLICIES="${POLICIES} reset"
 pol_reset() {
 	local label service
 	local endpoint_pre endpoint_post
+	local restarts_pre restarts_post
 
 	service=$1
 	label=$2
 
+	restarts_pre=$(get_value restarts ${service})
 	endpoint_pre=$(get_value endpoint ${service})
 
-	service refresh ${label}
+	service fi ${label}
-	wait_for_service ${service}
+	wait_for_service ${service} ${restarts_pre}
 
+	restarts_post=$(get_value restarts ${service})
 	endpoint_post=$(get_value endpoint ${service})
 
 	# This policy doesn't guarantee the endpoint to be kept, but there
 	# is a slight chance that it will actualy stay the same, and fail
 	# the test.
-	if [ ! ${endpoint_post} -eq ${endpoint_pre} ]
+	if [ ${restarts_post} -gt ${restarts_pre} \
+	    -a ${endpoint_post} -ne ${endpoint_pre} ]
 	then
 		echo ok
 	else
@@ -122,6 +127,138 @@ pol_reset() {
 	fi
 }
 
+#######################################################################
+# Live update test
+#######################################################################
+lu_test_one() {
+	local label=$1
+	local prog=$2
+	local result=$3
+	lu_opts=${lu_opts:-}
+	lu_maxtime=${lu_maxtime:-3HZ}
+	lu_state=${lu_state:-1}
+
+	service ${lu_opts} update ${prog} -label ${label} -maxtime ${lu_maxtime} -state ${lu_state}
+	if [ $? -ne $result ]
+	then
+		echo not ok
+		return 1
+	else
+		return 0
+	fi
+}
+
+lu_test() {
+	local label service
+	local endpoint_pre endpoint_post
+	local restarts_pre restarts_post
+
+	service=$1
+	label=$2
+
+	restarts_pre=$(get_value restarts ${service})
+	endpoint_pre=$(get_value endpoint ${service})
+
+	lu_test_one ${label} self 0 || return
+	if ! echo "vm pm vfs rs" | grep -q ${label}
+	then
+		lu_opts="-o" lu_test_one ${label} self 0 || return
+	fi
+	lu_opts="-x" lu_test_one ${label} self 200 || return
+	if ! echo "rs" | grep -q ${label}
+	then
+		lu_opts="-y" lu_test_one ${label} self 78 || return
+		lu_maxtime="1HZ" lu_opts="-z" lu_test_one ${label} self 4 || return
+	fi
+	lu_maxtime="1HZ" lu_state="5" lu_test_one ${label} self 4 || return
+
+	restarts_post=$(get_value restarts ${service})
+	endpoint_post=$(get_value endpoint ${service})
+
+	# Make sure endpoint and restarts are preserved
+	if [ ${restarts_post} -eq ${restarts_pre} \
+	    -a ${endpoint_post} -eq ${endpoint_pre} ]
+	then
+		echo ok
+	else
+		echo not ok
+	fi
+}
+
+multi_lu_test_one() {
+	local result=$1
+	shift
+	local labels="$*"
+	local ret=0
+	local index=0
+	local once_index=1
+
+	lu_opts=${lu_opts:-}
+	lu_maxtime=${lu_maxtime:-3HZ}
+	lu_state=${lu_state:-1}
+	lu_opts_once=${lu_opts_once:-$lu_opts}
+	lu_maxtime_once=${lu_maxtime_once:-$lu_maxtime}
+	lu_state_once=${lu_state_once:-$lu_state}
+
+	for label in ${labels}
+	do
+		if [ $index -eq $once_index ]
+		then
+			service ${lu_opts_once} -q update self -label ${label} -maxtime ${lu_maxtime_once} -state ${lu_state_once} || ret=1
+		else
+			service ${lu_opts} -q update self -label ${label} -maxtime ${lu_maxtime} -state ${lu_state} || ret=1
+		fi
+		index=`expr $index + 1`
+	done
+	service sysctl upd_run
+	if [ $? -ne $result ]
+	then
+		ret=1
+	fi
+	if [ $ret -eq 1 ]
+	then
+		echo not ok
+	fi
+	return $ret
+}
+
+multi_lu_test() {
+	local labels="$*"
+
+	multi_lu_test_one 0 ${labels} || return
+	lu_opts_once="-x" multi_lu_test_one 200 ${labels} || return
+	lu_opts_once="-y" multi_lu_test_one 200 ${labels} || return
+	lu_maxtime_once="1HZ" lu_opts_once="-z" multi_lu_test_one 200 ${labels} || return
+	lu_maxtime_once="1HZ" lu_state_once="5" multi_lu_test_one 4 ${labels} || return
+
+	echo ok
+}
+
+#######################################################################
+# Service management routines
+#######################################################################
+prepare_service() {
+	local label service
+
+	service=$1
+	label=$2
+
+	flags=$(get_value flags ${service})
+	echo $flags | grep -q 'r' || return 0
+	echo $flags | grep -q 'R' && return 0
+
+	service clone $label
+	return 1
+}
+
+cleanup_service() {
+	local label
+
+	label=$1
+
+	service unclone $label
+}
+
 #######################################################################
 # main()
 #######################################################################
@@ -136,27 +273,65 @@ main() {
 		exit 1
 	fi
 
-	labels=$(echo /proc/service/*)
+	services=$(echo /proc/service/*)
-	for label in ${labels}
+	for service in ${services}
 	do
-		service_policies=$(grep policies ${label}|cut -d: -f2)
+		label=$(basename ${service})
+		service_policies=$(grep policies ${service}|cut -d: -f2)
 		for pol in ${service_policies}
 		do
 			# Check if the supported policy is under test
 			if echo "${POLICIES}" | grep -q ${pol}
 			then
 				echo "# testing ${label} :: ${pol}"
-				result=$(pol_${pol} ${label} $(basename ${label}))
+				cleanup=0
-				#pol_${pol} ${label} $(basename ${label})
+				prepare_service ${service} ${label} || cleanup=1
-				#result="FAILED"
+				result=$(pol_${pol} ${service} ${label})
 				if [ "x${result}" != "xok" ]
 				then
 					echo "not ok # failed ${label}, ${pol}"
 					exit 1
 				fi
+				if [ $cleanup -eq 1 ]
+				then
+					cleanup_service ${label}
+				fi
 			fi
 		done
 	done
+	multi_lu_labels=""
+	for service in ${services}
+	do
+		label=$(basename ${service})
+		service_policies=$(grep policies ${service}|cut -d: -f2)
+		if echo "${service_policies}" | grep -q "[a-zA-Z]"
+		then
+			echo "# testing ${label} :: live update+rollback"
+			result=$(lu_test ${service} ${label})
+			if [ "x${result}" != "xok" ]
+			then
+				echo "not ok # failed ${label}, live update+rollback"
+				exit 1
+			fi
+			if [ "x${label}" = "xrs" ]
+			then
+				continue
+			fi
+			service_flags=$(get_value flags ${service})
+			if echo $service_flags | grep -q 's'
+			then
+				multi_lu_labels="${multi_lu_labels} ${label}"
+			fi
+		fi
+	done
+	multi_lu_labels="${multi_lu_labels} rs"
+	echo "# testing ${multi_lu_labels} :: whole-OS live update+rollback"
+	result=$(multi_lu_test $multi_lu_labels)
+	if [ "x${result}" != "xok" ]
+	then
+		echo "not ok # failed whole-OS live update+rollback"
+		exit 1
+	fi
 
 	echo ok
 	exit 0