tests: extend multicomponent live update test

- test multicomponent live update with and without rs and/or vm;
- retry the update a few times if the failure code suggests it might
  be a transient failure.

Change-Id: I5fce256bb418be257353ed21428f672d851d974d

minix/tests/testrelpol.sh

@@ -42,6 +42,7 @@
 #######################################################################
 POLICIES=""
 MAX_RETRY=7 # so that a single test takes at most 10 seconds
+MAX_MULTI_LU_RETRY=3 # how many times should we retry after bad luck?
 
 # get_value(key, filename, noerror)
 get_value() {
@@ -246,12 +247,13 @@ lu_test() {
 }
 
 multi_lu_test_one() {
-	local result=$1
+	local expected=$1
-	shift
+	local once_index=$2
+	shift 2
 	local labels="$*"
-	local ret=0
+	local ret=1
-	local index=0
+	local retry=0
-	local once_index=2
+	local index result
 
 	lu_opts=${lu_opts:-}
 	lu_maxtime=${lu_maxtime:-3HZ}
@@ -260,39 +262,87 @@ multi_lu_test_one() {
 	lu_maxtime_once=${lu_maxtime_once:-$lu_maxtime}
 	lu_state_once=${lu_state_once:-$lu_state}
 
-	for label in ${labels}
+	while [ $ret -eq 1 -a $retry -lt ${MAX_MULTI_LU_RETRY} ]
 	do
-		index=`expr $index + 1`
+		index=0
+		for label in ${labels}
+		do
+			index=`expr $index + 1`
 
-		if [ $index -eq $once_index ]
+			if [ $index -eq $once_index ]
+			then
+				service ${lu_opts_once} -q update self \
+					-label ${label} \
+					-maxtime ${lu_maxtime_once} \
+					-state ${lu_state_once} || ret=2
+			else
+				service ${lu_opts} -q update self \
+					-label ${label} \
+					-maxtime ${lu_maxtime} \
+					-state ${lu_state} || ret=2
+			fi
+		done
+		service sysctl upd_run
+		result=$?
+
+		# We may experience transient failures as a result of services
+		# trying to talk to each other while being prepared for the
+		# live update.  In that case we get result code 4.  If that is
+		# not the result code we expected, try again for a limited
+		# number of times.
+		if [ $result -eq $expected ]
 		then
-			service ${lu_opts_once} -q update self -label ${label} -maxtime ${lu_maxtime_once} -state ${lu_state_once} || ret=1
+			ret=0
-		else
+		elif [ $result -ne 4 ]
-			service ${lu_opts} -q update self -label ${label} -maxtime ${lu_maxtime} -state ${lu_state} || ret=1
+		then
+			break
 		fi
+		retry=`expr $retry + 1`
 	done
-	service sysctl upd_run
+
-	if [ $? -ne $result ]
-	then
-		ret=1
-	fi
-	if [ $ret -eq 1 ]
-	then
-		echo not ok
-	fi
 	return $ret
 }
 
 multi_lu_test() {
+	local y_result z_result
+	local have_rs=0
 	local labels="$*"
 
-	multi_lu_test_one 0 ${labels} || return
+	# Some of the results depend on whether RS is part of the live update.
-	lu_opts_once="-x" multi_lu_test_one 200 ${labels} || return
+	for label in ${labels}
-	lu_opts_once="-y" multi_lu_test_one 200 ${labels} || return
+	do
-	lu_maxtime_once="1HZ" lu_opts_once="-z" multi_lu_test_one 200 ${labels} || return
+		if [ "x$label" = "xrs" ]
-	lu_maxtime_once="1HZ" lu_state_once="5" multi_lu_test_one 4 ${labels} || return
+		then
+			have_rs=1
+		fi
+	done
 
-	echo ok
+	if [ $have_rs -eq 1 ]
+	then
+		y_result=200
+		z_result=200
+	else
+		y_result=78
+		z_result=4
+	fi
+
+	multi_lu_test_one 0 0 ${labels} || return 1
+	lu_opts_once="-x" multi_lu_test_one 200 2 ${labels} || return 1
+	lu_opts_once="-y" multi_lu_test_one ${y_result} 3 ${labels} || return 1
+	lu_maxtime_once="1HZ" lu_opts_once="-z" multi_lu_test_one ${z_result} 2 ${labels} || return 1
+	lu_maxtime_once="1HZ" lu_state_once="5" multi_lu_test_one 4 3 ${labels} || return 1
+
+	return 0
+}
+
+multi_lu_test_wrapper() {
+	echo "# testing $@ :: multicomponent live update+rollback"
+	if ! multi_lu_test "$@"
+	then
+		echo "not ok # failed multicomponent live update+rollback"
+		return 1
+	fi
+	return 0
 }
 
 #######################################################################
@@ -361,7 +411,7 @@ main() {
 				echo "not ok # failed ${label}, live update+rollback"
 				exit 1
 			fi
-			if [ "x${label}" = "xrs" ]
+			if [ "x${label}" = "xrs" -o "x${label}" = "xvm" ]
 			then
 				continue
 			fi
@@ -372,14 +422,11 @@ main() {
 			fi
 		fi
 	done
-	multi_lu_labels="${multi_lu_labels} rs"
+
-	echo "# testing ${multi_lu_labels} :: whole-OS live update+rollback"
+	multi_lu_test_wrapper ${multi_lu_labels} || exit 1
-	result=$(multi_lu_test $multi_lu_labels)
+	multi_lu_test_wrapper ${multi_lu_labels} vm || exit 1
-	if [ "x${result}" != "xok" ]
+	multi_lu_test_wrapper ${multi_lu_labels} rs || exit 1
-	then
+	multi_lu_test_wrapper ${multi_lu_labels} vm rs || exit 1
-		echo "not ok # failed whole-OS live update+rollback"
-		exit 1
-	fi
 
 	echo ok
 	exit 0