Basic live rerandomization infrastructure

This commits adds a basic infrastructure to support Address Space
Randomization (ASR).  In a nutshell, using the already imported ASR
LLVM pass, multiple versions can be generated for the same system
service, each with a randomized, different address space layout.
Combined with the magic instrumentation for state transfer, a system
service can be live updated into another ASR-randomized version at
runtime, thus providing live rerandomization.

Since MINIX3 is not yet capable of running LLVM linker passes, the
ASR-randomized service binaries have to be pregenerated during
crosscompilation.  These pregenerated binaries can then be cycled
through at runtime.  This patch provides the basic proof-of-concept
infrastructure for both these parts.

In order to support pregeneration, the clientctl host script has
been extended with a "buildasr" command.  It is to be used after
building the entire system with bitcode and magic support, and will
produce a given number of ASR-randomized versions of all system
services.  These services are placed in /usr/service/asr in the
image that is generated as final step by the "buildasr" command.

In order to support runtime updating, a new update_asr(8) command
has been added to MINIX3.  This command attempts to live-update the
running system services into their next ASR-randomized versions.
For now, this command is not run automatically, and thus must be
invoked manually.

Technical notes:

- For various reasons, magic instrumentation is x86-only for now,
  and ASR functionality is therefore to be used on x86 only as well.
- The ASR-randomized binaries are placed in numbered subdirectories
  so as not to have to change their actual program names, which are
  assumed to be static in various places (system.conf, procfs).
- The root partition is typically too small to contain all the
  produced binaries, which is why we introduce /usr/service.  There
  is a symlink from /service/asr to /usr/service/asr for no other
  reason than to let userland continue to assume that all services
  are reachable through /service.
- The ASR count field (r_asr_count/ASRcount) maintained by RS is not
  used within RS in any way; it is only passed through procfs to
  userland in order to allow update_asr(8) to keep track of which
  version is currently loaded without having to maintain own state.
- Ideally, pre-instrumentation linking of a service would remove all
  its randomized versions.  Currently, the user is assumed not to
  perform ASR instrumentation and then recompile system services
  without performing ASR instrumentation again, as the randomized
  binaries included in the image would then be stale.  This aspect
  has to be improved later.
- Various other issues are flagged in the comments of the various
  parts of this patch.

Change-Id: I093ad57f31c18305591f64b2d491272288aa0937

distrib/sets/checkflist

@@ -100,6 +100,7 @@ shift $((${OPTIND} - 1))
 # * ignore var/db/syspkg and its contents
 # * ignore METALOG and METALOG.*
 # * ignore etc/mtree/set.*
+# * MINIX3 only: ignore ASR service binaries
 #
 ignore_exceptions()
 {
@@ -107,12 +108,14 @@ IGNORE_REGEXP_SYSPKG="^\./var/db/syspkg(\$|/)"
 IGNORE_REGEXP_METALOG="^\./METALOG(\..*)?\$"
 IGNORE_REGEXP_SETS="^\./SETS\..*\$"
 IGNORE_REGEXP_MTREE="^\./etc/mtree/set\.[a-z]*\$"
+IGNORE_REGEXP_SERVICE_ASR="^\./usr/service/asr/"
 
 	${EGREP} -v \
 		-e "${IGNORE_REGEXP_SYSPKG}" \
 		-e "${IGNORE_REGEXP_METALOG}" \
 		-e "${IGNORE_REGEXP_SETS}" \
-		-e "${IGNORE_REGEXP_MTREE}"
+		-e "${IGNORE_REGEXP_MTREE}" \
+		-e "${IGNORE_REGEXP_SERVICE_ASR}"
 }
 
 #

distrib/sets/lists/minix/mi

@@ -184,6 +184,7 @@
 ./sbin/reboot				minix-sys
 ./sbin/shutdown				minix-sys
 ./service				minix-sys
+./service/asr				minix-sys
 ./service/devman			minix-sys
 ./service/ds				minix-sys
 ./service/emmc				minix-sys
@@ -5269,6 +5270,7 @@
 ./usr/man/man8/unstr.8				minix-sys
 ./usr/man/man8/update.8				minix-sys
 ./usr/man/man8/updateboot.8		minix-sys
+./usr/man/man8/update_asr.8			minix-sys
 ./usr/man/man8/update_bootcfg.8		minix-sys
 ./usr/man/man8/usage.8				minix-sys
 ./usr/man/man8/user.8				minix-sys
@@ -5318,6 +5320,7 @@
 ./usr/sbin/syslogd			minix-sys
 ./usr/sbin/traceroute			minix-sys
 ./usr/sbin/unlink			minix-sys
+./usr/sbin/update_asr			minix-sys
 ./usr/sbin/user				minix-sys
 ./usr/sbin/useradd			minix-sys
 ./usr/sbin/userdel			minix-sys
@@ -5327,6 +5330,8 @@
 ./usr/sbin/vnconfig			minix-sys
 ./usr/sbin/vndconfig			minix-sys
 ./usr/sbin/zic				minix-sys
+./usr/service				minix-sys
+./usr/service/asr			minix-sys
 ./usr/share				minix-sys
 ./usr/share/atf				minix-sys	atf
 ./usr/share/atf/atf-run.hooks		minix-sys	atf,!kyua

etc/Makefile

@@ -275,7 +275,8 @@ CONFIGSYMLINKS+=	${TZDIR}/${LOCALTIME}	/etc/localtime \
 CONFIGSYMLINKS+=	\
 			/usr/log		/var/log \
 			/usr/tmp		/var/tmp \
-			/proc/mounts		/etc/mtab
+			/proc/mounts		/etc/mtab \
+			/usr/service/asr	/service/asr
 
 .endif # !defined(__MINIX)
 

etc/mtree/NetBSD.dist.base

@@ -120,6 +120,8 @@
 ./usr/preserve
 ./usr/run
 ./usr/sbin
+./usr/service
+./usr/service/asr
 ./usr/share
 ./usr/share/calendar
 ./usr/share/doc

minix/commands/Makefile

@@ -30,7 +30,7 @@ SUBDIR=	add_route arp at backup btrace \
 	update version vol \
 	writeisofs fetch \
 	zdump zmodem pkgin_cd pkgin_all pkgin_sets \
-	worldstone updateboot update_bootcfg \
+	worldstone updateboot update_asr update_bootcfg \
 	atnormalize dosread fdisk loadfont \
 	autopart part partition playwave  \
 	recwave repartition screendump

minix/commands/service/service.c

@@ -136,6 +136,7 @@ static int known_request_types[] = {
 #define ARG_TRG_LABELNAME "-trg-label"	/* target label name */
 #define ARG_LU_IPC_BL	"-ipc_bl"       /* IPC blacklist filter */
 #define ARG_LU_IPC_WL	"-ipc_wl"       /* IPC whitelist filter */
+#define ARG_ASR_COUNT	"-asr-count"    /* number of ASR live updates */
 #define ARG_RESTARTS	"-restarts"    /* number of restarts */
 
 /* The function parse_arguments() verifies and parses the command line 
@@ -158,6 +159,7 @@ static int custom_config_file = 0;
 static int req_lu_state = DEFAULT_LU_STATE;
 static int req_lu_maxtime = DEFAULT_LU_MAXTIME;
 static int req_restarts = 0;
+static int req_asr_count = -1;
 static long req_heap_prealloc = 0;
 static long req_map_prealloc = 0;
 static int req_sysctl_type = 0;
@@ -175,7 +177,7 @@ static void print_usage(char *app_name, char *problem)
   fprintf(stderr, "Warning, %s\n", problem);
   fprintf(stderr, "Usage:\n");
   fprintf(stderr,
-      "    %s [%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s] (up|run|edit|update) <binary|%s> [%s <args>] [%s <special>] [%s <major_nr>] [%s <dev_id>] [%s <ticks>] [%s <path>] [%s <name>] [%s <path>] [%s <state value|eval_expression>] [%s <time>] [%s <bytes>] [%s <bytes>] [%s <name>] [(%s|%s <src_label1,src_type1:src_label2,:,src_type3:...>)*] [%s <restarts>]\n",
+      "    %s [%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s] (up|run|edit|update) <binary|%s> [%s <args>] [%s <special>] [%s <major_nr>] [%s <dev_id>] [%s <ticks>] [%s <path>] [%s <name>] [%s <path>] [%s <state value|eval_expression>] [%s <time>] [%s <bytes>] [%s <bytes>] [%s <name>] [(%s|%s <src_label1,src_type1:src_label2,:,src_type3:...>)*] [%s <count>] [%s <restarts>]\n",
 	app_name, OPT_COPY, OPT_REUSE, OPT_NOBLOCK, OPT_REPLICA, OPT_NO_BIN_EXP,
 	OPT_BATCH, OPT_ASR_LU, OPT_PREPARE_ONLY_LU, OPT_FORCE_SELF_LU,
 	OPT_FORCE_INIT_CRASH, OPT_FORCE_INIT_FAIL, OPT_FORCE_INIT_TIMEOUT,
@@ -184,7 +186,7 @@ static void print_usage(char *app_name, char *problem)
 	ARG_ARGS, ARG_DEV, ARG_MAJOR, ARG_DEVMANID, ARG_PERIOD,
 	ARG_SCRIPT, ARG_LABELNAME, ARG_CONFIG, ARG_LU_STATE, ARG_LU_MAXTIME,
 	ARG_HEAP_PREALLOC, ARG_MAP_PREALLOC, ARG_TRG_LABELNAME, ARG_LU_IPC_BL, ARG_LU_IPC_WL,
-	ARG_RESTARTS);
+	ARG_ASR_COUNT, ARG_RESTARTS);
   fprintf(stderr, "    %s down <label>\n", app_name);
   fprintf(stderr, "    %s refresh <label>\n", app_name);
   fprintf(stderr, "    %s restart <label>\n", app_name);
@@ -629,6 +631,14 @@ static int parse_arguments(int argc, char **argv, u32_t *rss_flags)
                   exit(r);
               }
           }
+          else if (strcmp(argv[i], ARG_ASR_COUNT)==0) {
+              errno=0;
+              req_asr_count = strtol(argv[i+1], &buff, 10);
+              if(errno || strcmp(buff, "") || req_asr_count<0) {
+                  print_usage(argv[ARG_NAME], "bad ASR count");
+                  exit(EINVAL);
+              }
+          }
           else if (strcmp(argv[i], ARG_RESTARTS)==0) {
               errno=0;
               req_restarts = strtol(argv[i+1], &buff, 10);
@@ -746,6 +756,7 @@ int main(int argc, char **argv)
       config.rs_start.rss_major= req_major;
       config.rs_start.rss_period= req_period;
       config.rs_start.rss_script= req_script;
+      config.rs_start.rss_asr_count= req_asr_count;
       config.rs_start.rss_restarts= req_restarts;
       config.rs_start.devman_id= devman_id;
       config.rs_start.rss_heap_prealloc_bytes= req_heap_prealloc;

minix/commands/update_asr/Makefile

@@ -0,0 +1,5 @@
+SCRIPTS=	update_asr.sh
+MAN=		update_asr.8
+BINDIR=		/usr/sbin
+
+.include <bsd.prog.mk>

minix/commands/update_asr/update_asr.8

@@ -0,0 +1,51 @@
+.Dd September 7, 2015
+.Dt UPDATE_ASR 8
+.Os
+.Sh NAME
+.Nm update_asr
+.Nd perform ASR rerandomization on system services
+.Sh SYNOPSYS
+.Nm
+.Op Fl v
+.Op Ar labels
+.Sh DESCRIPTION
+The
+.Nm
+utility performs one cycle of system service live
+ASR (Address Space Randomization) rerandomization.
+By default, the utility will attempt to update all system services.
+If a space-separated list of service
+.Ar labels
+is given, only those services are updated.
+.Pp
+Updates require the presence of at least two precreated ASR binaries for the
+service: the original service binary, and at least one rerandomized ASR binary
+for the service.
+The update consists of selecting the next on-disk ASR binary for the service,
+and performing a live update from the current service into the selected new
+version.
+The selection takes place in a round-robin fashion, so once the script has
+gone through all rerandomized ASR binaries, it will revert to the original
+service binary, and then continue with the first rerandomized ASR binary
+again, and so on.
+.Pp
+The following options are available:
+.Bl -tag -width Ds
+.It Fl v
+Enable verbose mode.
+.El
+.Sh SEE ALSO
+.Xr service 8
+.Sh AUTHORS
+The
+.Nm
+utility was written by
+.An David van Moolenbroek
+.Aq david@minix3.org .
+.Sh BUGS
+Failures are silently ignored.
+Some failures are expected, since not all services are necessarily quiescent
+and therefore ready to be updated.
+.Pp
+As of writing, no infrastructure exists to perform ASR updates automatically,
+and no infrastructure exists to create new rerandomized binaries at runtime.

minix/commands/update_asr/update_asr.sh

@@ -0,0 +1,135 @@
+#!/bin/sh
+# ASR live update script by David van Moolenbroek <david@minix3.org>
+
+# The path to the initial, standard system service binaries.
+SERVICE_PATH=/service
+
+# The path to the alternative, ASR-rerandomized system service binaries.
+# The path used here is typically a symlink into /usr for size reasons.
+# As of writing, the only way to create these sets of binaries is by means
+# of the host-side "minix/llvm/clientctl buildasr" command.
+SERVICE_ASR_PATH=$SERVICE_PATH/asr
+
+# A space-separated list of labels not to update in any case.  The list
+# includes the memory service, which is currently not compiled with bitcode
+# and therefore also not instrumented.  It also contains the VM service,
+# for which ASR is possible but too dangerous: much of its address space is
+# deliberately ignored by the instrumentation, and ASR would invalidate any
+# pointers from the ignored memory to the relocated memory.  Note that
+# skipped services may still have rerandomized binaries on disk.
+SKIP="memory vm"
+
+# Custom live update states to use for certain label prefixes.  This list is
+# made up of space-separated tokens, each token consisting of a label prefix,
+# followed by a colon, followed by the state number to use for those labels.
+# Currently it contains all services that make use of worker threads.  This
+# setting should not need to exist; see the corresponding TODO item below.
+STATES="vfs:2 ahci_:2 virtio_blk_:2"
+
+# If this variable is set, it is used as timeout for the live updates.  The
+# service(8) argument takes a number of click ticks, or a number of seconds
+# if the value ends with "HZ".
+TIMEOUT=300HZ
+
+# Configuration ends here.
+
+debug() {
+	if [ $verbose -eq 1 ]; then
+		echo "$@"
+	fi
+}
+
+verbose=0
+ret=0
+
+while getopts 'v' opt; do
+	case $opt in
+	v)	verbose=1
+		;;
+	?)	echo "Usage: $0 [-v] [label [label..]]" >&2
+		exit 1
+	esac
+done
+shift $(($OPTIND - 1))
+
+if [ $# -eq 0 ]; then
+	services=$(echo /proc/service/*)
+else
+	services="$@"
+fi
+
+for service in $services; do
+	label=$(basename $service)
+	filename=$(grep filename: $service | cut -d' ' -f2)
+	count=$(grep ASRcount: $service | cut -d' ' -f2)
+
+	# Start by making sure we are not supposed to skip this service.
+	if echo " $SKIP " | grep -q " $label "; then
+		debug "skipping $label: found in skip list"
+		continue
+	fi
+
+	# The base binary of the program has number 0 and must be present.
+	if [ ! -f $SERVICE_PATH/$filename ]; then
+		debug "skipping $label: no base binary found"
+		continue
+	fi
+
+	# Count the ASR binaries for this program, starting from number 1.
+	# There must be at least one, so that we can switch between versions.
+	# By counting using a number rather than using a directory iterator,
+	# we avoid potential problems with gaps between the numbers by
+	# stopping at the first number for which no binary is present.
+	total=1
+	while [ -f $SERVICE_ASR_PATH/$total/$filename ]; do
+		total=$(($total + 1))
+	done
+
+	if [ $total -eq 1 ]; then
+		debug "skipping $label: no ASR binaries found"
+		continue
+	fi
+
+	# Determine the path name of the binary to use for this update.
+	# TODO: pick the next binary at random rather than round-robin.
+	count=$((($count + 1) % $total))
+	if [ $count -eq 0 ]; then
+		binary=$SERVICE_PATH/$filename
+	else
+		binary=$SERVICE_ASR_PATH/$count/$filename
+	fi
+
+	# Check whether the live update should use a state other than the
+	# default (namely state 1, which is "work free").  In particular, any
+	# programs that use threads typically need another state (namely state
+	# 2, which is "request free".  TODO: allow services to specify their
+	# own default state, thus avoiding the redundancy introduced here.
+	state=
+	for token in $STATES; do
+		prefix=$(echo $token | cut -d: -f1)
+		if echo "$label" | grep -q -e "^$prefix"; then
+			state="-state $(echo $token | cut -d: -f2)"
+		fi
+	done
+
+	# Apply a custom timeout if present.  This may be necessary in VMs.
+	maxtime=
+	if [ -n "$TIMEOUT" ]; then
+		maxtime="-maxtime $TIMEOUT"
+	fi
+
+	# Perform the live update.  The update may legitimately fail if the
+	# service is not in the right state.  TODO: report transient errors
+	# as debugging output only.
+	service -a update $binary -label $label -asr-count $count \
+		$state $maxtime
+	error=$?
+	if [ $error -eq 0 ]; then
+		debug "updated $label to number $count, total $total"
+	else
+		echo "failed updating $label: error $error" >&2
+		ret=1
+	fi
+done
+
+exit $ret

minix/fs/procfs/service.c

@@ -338,4 +338,5 @@ service_read(struct inode * node)
 	buf_printf("restarts: %d\n", rp->r_restarts);
 	buf_printf("flags:    %s\n", service_get_flags(slot));
 	buf_printf("policies: %s\n", service_get_policies(policies, slot));
+	buf_printf("ASRcount: %u\n", rp->r_asr_count);
 }

minix/include/minix/rs.h

@@ -113,6 +113,7 @@ struct rs_start
 	long rss_period;
 	char *rss_script;
 	size_t rss_scriptlen;
+	long rss_asr_count;
 	long rss_restarts;
 	long rss_heap_prealloc_bytes;
 	long rss_map_prealloc_bytes;

minix/llvm/clientctl

@@ -371,6 +371,52 @@ function minix_bisect {
     cd -
 }
 
+# Usage: [C=set] ./clientctl buildasr [num]
+#
+# Build 'num' sets of ASR-randomized service binaries for the 'set' set of
+# services.  Defaults to one set (in addition to the set used to boot) for
+# all services.  To be used after building the full system.
+#
+# The MINIX3 counterpart of the generation taking place here is the
+# update_asr(8) command, which cycles through the generated binaries.
+#
+function minix_buildasr {
+    MINIXLLVMDIR=$ROOT/minix/llvm
+    ASRDIR=/usr/service/asr
+
+    . $MINIXLLVMDIR/common.inc		# get DESTDIR
+    DESTDIR="$DESTDIR/destdir.i386"	# correct DESTDIR
+
+    ASRDESTDIR="$DESTDIR$ASRDIR"
+    COUNT=${1:-1}	# take count from command line, default to 1
+    C=${C:-"servers,fs,net,drivers"}
+
+    # start by relinking everything against the magic library
+    C=$C $MINIXLLVMDIR/relink.llvm magic
+
+    # we are replacing any previously made ASR binaries
+    rm -rf $ASRDESTDIR/*
+
+    # generate $COUNT number of sets of ASR-randomized service binaries
+    # TODO: do not use current time as random seed
+    N=1
+    while [ $N -le $COUNT ]; do
+        mkdir $ASRDESTDIR/$N
+        export BINDIR=$ASRDIR/$N
+        C=$C $MINIXLLVMDIR/build.llvm magic asr
+        sleep 1 # just to make sure they're guaranteed to be different
+        N=$(($N + 1))
+    done
+
+    # generate the initial set of service binaries, different as well
+    unset BINDIR
+    C=$C $MINIXLLVMDIR/build.llvm magic asr
+
+    # finally generate the image
+    # x86_hdimage will automatically add the binaries to the image set
+    $MINIXLLVMDIR/clientctl buildimage
+}
+
 case "$mode" in
   'buildimage')
     (cd $ROOT && CREATE_IMAGE_ONLY=1 releasetools/x86_hdimage.sh -b)
@@ -409,6 +455,9 @@ case "$mode" in
   'bisect')
     minix_bisect
     ;;
+  'buildasr')
+    minix_buildasr $*
+    ;;
   *)
     echo "Invalid action: $mode"
     exit 1

minix/servers/rs/main.c

@@ -331,6 +331,7 @@ static int sef_cb_init_fresh(int UNUSED(type), sef_init_info_t *UNUSED(info))
       rp->r_check_tm = 0;                      /* not checked yet */
       getticks(&rp->r_alive_tm);               /* currently alive */
       rp->r_stop_tm = 0;                       /* not exiting yet */
+      rp->r_asr_count = 0;                     /* no ASR updates yet */
       rp->r_restarts = 0;                      /* no restarts so far */
       rp->r_period = 0;                        /* no period yet */
       rp->r_exec = NULL;                       /* no in-memory copy yet */

minix/servers/rs/manager.c

@@ -1690,6 +1690,11 @@ endpoint_t source;
       rp->r_restarts = rs_start->rss_restarts;
   }
 
+  /* Update number of ASR live updates. */
+  if(rs_start->rss_asr_count >= 0) {
+      rp->r_asr_count = rs_start->rss_asr_count;
+  }
+
   /* (Re)initialize privilege settings. */
   init_privs(rp, &rp->r_priv);
 
@@ -1760,6 +1765,7 @@ endpoint_t source;
   }
   
   /* Initialize some fields. */
+  rp->r_asr_count = 0;				/* no ASR updates yet */
   rp->r_restarts = 0; 				/* no restarts yet */
   rp->r_old_rp = NULL;			        /* no old version yet */
   rp->r_new_rp = NULL;			        /* no new version yet */

minix/servers/rs/type.h

@@ -62,6 +62,7 @@ struct rproc {
   struct rprocupd r_upd;        /* update descriptor */
   pid_t r_pid;			/* process id, -1 if the process is not there */
 
+  int r_asr_count;		/* number of live updates with ASR */
   int r_restarts;		/* number of restarts (initially zero) */
   long r_backoff;		/* number of periods to wait before revive */
   unsigned r_flags; 		/* status and policy flags */

releasetools/x86_hdimage.sh

@@ -160,6 +160,11 @@ fi
 # add fstab
 echo "./etc/fstab type=file uid=0 gid=0 mode=0755 size=747 time=1365060731.000000000" >> ${IMG_DIR}/input
 
+# add any generated ASR-randomized service binaries (but not their root directory, which is already there)
+# TODO: apply stricter file permissions for both these and the base /service binaries, against local attacks
+(cd ${DESTDIR} && find ./usr/service/asr -type d | sed '1d;s/$/ type=dir uid=0 gid=0 mode=0755/') >> ${IMG_DIR}/input
+(cd ${DESTDIR} && find ./usr/service/asr -type f | sed 's/$/ type=file uid=0 gid=0 mode=0755/') >> ${IMG_DIR}/input
+
 # fill root.img (skipping /usr entries while keeping the /usr directory)
 cat ${IMG_DIR}/input  | grep -v "^./usr/" | ${CROSS_TOOLS}/nbtoproto -b ${DESTDIR} -o ${IMG_DIR}/root.proto