x86 hd image creator

A script that creates a HD image ready to be booted by an x86
emulator, ready to work in a crossbuild environment.

It's really just for qemu/kvm as there's no boot code in
the MBR and no bootloader installed so we rely on the in-kvm
multiboot implementation for now. This is very convenient for
passing args too.

To minimize reliance on external tools, we use the Minix
'partition' utility to write the partition table of the HD
image, which therefore has to be compiled natively.

	. new script releasetools/x86_hdimage.sh
	. natively compile minix 'partition' utility
	. make <machine/partition.h> 64-bit safe for it

Change-Id: If645b4691536752271e0b8a8ed59a34f248dace4

commands/Makefile.inc

@@ -1,4 +1,10 @@
+
+# Only include Minix-specific libraries when compiling
+# for non-Minix, i.e. not natively in a crossbuild.
+.if !defined(HOSTPROGNAME)
 LDADD+= -lminlib -lcompat_minix -lasyn -lterminfo
+.endif
+
 BINDIR?=/usr/bin
 
 # BJG too many warnings here

commands/partition/Makefile

@@ -1,4 +1,8 @@
 PROG=	partition
 MAN=
 
+# We need this to find our partition.h while compiling natively
+# on non-Minix.
+CPPFLAGS+= -I${NETBSDSRCDIR}/include/arch/${MACHINE_ARCH}/include
+
 .include <bsd.prog.mk>

commands/partition/partition.c

@@ -7,16 +7,24 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <fcntl.h>
-#include <minix/config.h>
-#include <minix/const.h>
-#include <minix/partition.h>
-#include <minix/u64.h>
-#include <machine/partition.h>
 #include <sys/stat.h>
 #include <string.h>
 #include <errno.h>
 #include <sys/ioctl.h>
 #include <limits.h>
+#include <stdint.h>
+#include <assert.h>
+
+#ifdef __minix
+#include <machine/partition.h>
+#include <minix/config.h>
+#include <minix/const.h>
+#include <minix/partition.h>
+#include <minix/u64.h>
+#else
+#include "partition.h"
+#define NR_PARTITIONS 4
+#endif
 
 #define SECTOR_SIZE	512
 
@@ -59,10 +67,10 @@ int npart;
 void find_exist(struct part_entry *exist, int sysind, int nr)
 {
 	int f;
-	u16_t signature;
+	uint16_t signature;
 	struct part_entry oldtable[NR_PARTITIONS];
 	int n, i;
-	u32_t minlow, curlow;
+	uint32_t minlow, curlow;
 	struct part_entry *cur;
 	char *nr_s[] = { "", "second ", "third ", "fourth" };
 
@@ -109,7 +117,7 @@ void find_exist(struct part_entry *exist, int sysind, int nr)
 void write_table(void)
 {
 	int f;
-	u16_t signature= 0xAA55;
+	uint16_t signature= 0xAA55;
 	struct part_entry newtable[NR_PARTITIONS];
 	int i;
 
@@ -120,6 +128,14 @@ void write_table(void)
 
 	for (i= 0; i < NR_PARTITIONS; i++) newtable[i]= table[1 + 2*i];
 
+	/* we have a abstract struct but it must conform to a certain
+	 * reality that will never change (in-MBR sizes and offsets).
+	 * each partition entry is 16 bytes and there are 4 of them.
+	 * this also determines the signature offset.
+	 */
+	assert(sizeof(struct part_entry) == 16);
+	assert(sizeof(newtable) == 64);
+
 	if ((f= open(device, O_WRONLY)) < 0
 
 		|| lseek(f, (off_t) PART_TABLE_OFF, SEEK_SET) == -1
@@ -285,11 +301,14 @@ void geometry(void)
  */
 {
 	int fd;
-	struct part_geom geometry;
 	struct stat sb;
 
 	if ((fd= open(device, O_RDONLY)) < 0) fatal(device);
 
+#ifdef __minix
+	struct part_geom geometry;
+
+
 	/* Get the geometry of the drive, and the device's base and size. */
 	if (ioctl(fd, DIOCGETP, &geometry) < 0)
 	{
@@ -303,12 +322,21 @@ void geometry(void)
 		geometry.cylinders= (sb.st_size-1)/SECTOR_SIZE/
 			(geometry.sectors*geometry.heads) + 1;
 	}
-	close(fd);
 	primary.lowsec= div64u(geometry.base, SECTOR_SIZE);
 	primary.size= div64u(geometry.size, SECTOR_SIZE);
 	cylinders= geometry.cylinders;
 	heads= geometry.heads;
 	sectors= geometry.sectors;
+#else
+	if (fstat(fd, &sb) < 0) fatal(device);
+	primary.lowsec= 0;
+	primary.size= sb.st_size / SECTOR_SIZE;
+	heads= 64;
+	sectors= 32;
+	cylinders= (sb.st_size-1) / SECTOR_SIZE / (sectors*heads) + 1;
+#endif
+
+	close(fd);
 
 	/* Is this a primary partition table?  If so then pad partitions. */
 	pad= (!mflag && primary.lowsec == 0);
@@ -346,7 +374,7 @@ void distribute(void)
 			if (pe->bootind & EXIST_FLAG) {
 				if (base > pe->lowsec) {
 					fprintf(stderr,
-	"%s: fixed partition %d is preceded by too big partitions/holes\n",
+	"%s: fixed partition %ld is preceded by too big partitions/holes\n",
 						arg0, ((pe - table) - 1) / 2);
 					exit(1);
 				}

include/arch/earm/include/partition.h

@@ -2,17 +2,19 @@
 #ifndef _PARTITION_H
 #define _PARTITION_H
 
+#include <stdint.h>
+
 struct part_entry {
-  unsigned char bootind;	/* boot indicator 0/ACTIVE_FLAG	 */
+  uint8_t bootind;	/* boot indicator 0/ACTIVE_FLAG	 */
-  unsigned char start_head;	/* head value for first sector	 */
+  uint8_t start_head;	/* head value for first sector	 */
-  unsigned char start_sec;	/* sector value + cyl bits for first sector */
+  uint8_t start_sec;	/* sector value + cyl bits for first sector */
-  unsigned char start_cyl;	/* track value for first sector	 */
+  uint8_t start_cyl;	/* track value for first sector	 */
-  unsigned char sysind;		/* system indicator		 */
+  uint8_t sysind;		/* system indicator		 */
-  unsigned char last_head;	/* head value for last sector	 */
+  uint8_t last_head;	/* head value for last sector	 */
-  unsigned char last_sec;	/* sector value + cyl bits for last sector */
+  uint8_t last_sec;	/* sector value + cyl bits for last sector */
-  unsigned char last_cyl;	/* track value for last sector	 */
+  uint8_t last_cyl;	/* track value for last sector	 */
-  unsigned long lowsec;		/* logical first sector		 */
+  uint32_t lowsec;		/* logical first sector		 */
-  unsigned long size;		/* size of partition in sectors	 */
+  uint32_t size;		/* size of partition in sectors	 */
 };
 
 #define ACTIVE_FLAG	0x80	/* value for active in bootind field (hd0) */

include/arch/i386/include/partition.h

@@ -2,17 +2,19 @@
 #ifndef _PARTITION_H
 #define _PARTITION_H
 
+#include <stdint.h>
+
 struct part_entry {
-  unsigned char bootind;	/* boot indicator 0/ACTIVE_FLAG	 */
+  uint8_t bootind;	/* boot indicator 0/ACTIVE_FLAG	 */
-  unsigned char start_head;	/* head value for first sector	 */
+  uint8_t start_head;	/* head value for first sector	 */
-  unsigned char start_sec;	/* sector value + cyl bits for first sector */
+  uint8_t start_sec;	/* sector value + cyl bits for first sector */
-  unsigned char start_cyl;	/* track value for first sector	 */
+  uint8_t start_cyl;	/* track value for first sector	 */
-  unsigned char sysind;		/* system indicator		 */
+  uint8_t sysind;		/* system indicator		 */
-  unsigned char last_head;	/* head value for last sector	 */
+  uint8_t last_head;	/* head value for last sector	 */
-  unsigned char last_sec;	/* sector value + cyl bits for last sector */
+  uint8_t last_sec;	/* sector value + cyl bits for last sector */
-  unsigned char last_cyl;	/* track value for last sector	 */
+  uint8_t last_cyl;	/* track value for last sector	 */
-  unsigned long lowsec;		/* logical first sector		 */
+  uint32_t lowsec;		/* logical first sector		 */
-  unsigned long size;		/* size of partition in sectors	 */
+  uint32_t size;		/* size of partition in sectors	 */
 };
 
 #define ACTIVE_FLAG	0x80	/* value for active in bootind field (hd0) */

releasetools/x86_hdimage.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+set -e
+
+: ${ARCH=i386}
+: ${OBJ=../obj.${ARCH}}
+: ${CROSS_TOOLS=${OBJ}/"tooldir.`uname -s`-`uname -r`-`uname -m`"/bin}
+: ${CROSS_PREFIX=${CROSS_TOOLS}/i586-elf32-minix-}
+: ${JOBS=1}
+: ${DESTDIR=${OBJ}/destdir.$ARCH}
+: ${FSTAB=${DESTDIR}/etc/fstab}
+: ${BUILDVARS=}
+: ${BUILDSH=build.sh}
+
+#
+# Directory where to store temporary file system images
+#
+: ${IMG_DIR=${OBJ}/img}
+: ${IMG=minix_x86.img}
+
+if [ ! -f ${BUILDSH} ]
+then	echo "Please invoke me from the root source dir, where ${BUILDSH} is."
+	exit 1
+fi
+
+export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH
+
+#
+# Artifacts from this script are stored in the IMG_DIR
+#
+mkdir -p $IMG_DIR
+
+#
+# Call build.sh using a sloppy file list so we don't need to remove the installed /etc/fstag
+#
+export CPPFLAGS=${FLAG}
+sh ${BUILDSH} -V SLOPPY_FLIST=yes -V MKBINUTILS=yes -V MKGCCCMDS=yes -j ${JOBS} -m ${ARCH} -O ${OBJ} -D ${DESTDIR} ${BUILDVARS} -U -u distribution
+
+#
+# This script creates a bootable image and should at some point in the future
+# be replaced by makefs.
+#
+# All sized are written in 512 byte blocks
+#
+# we create a disk image of about 2 gig's
+# for alignement reasons, prefer sizes which are multiples of 4096 bytes
+#
+: ${IMG_SIZE=$((   2*(2**30) / 512))}
+: ${ROOT_SIZE=$((  64*(2**20) / 512))}
+: ${HOME_SIZE=$(( 128*(2**20) / 512))}
+: ${USR_SIZE=$((1536*(2**20) / 512))}
+
+#
+# create a fstab entry in /etc this is normally done during the
+# setup phase on x86
+#
+cat >${FSTAB} <<END_FSTAB
+/dev/c0d0p1   /home   mfs     rw                      0       2
+/dev/c0d0p2   /usr    mfs     rw                      0       2
+END_FSTAB
+
+rm -f ${DESTDIR}/SETS.*
+
+${CROSS_TOOLS}/nbpwd_mkdb -V 0 -p -d ${DESTDIR} ${DESTDIR}/etc/master.passwd
+
+#
+# Now given the sizes above use DD to create separate files representing
+# the partitions we are going to use.
+#
+dd if=/dev/zero of=${IMG_DIR}/root.img bs=512 count=1 seek=$(($ROOT_SIZE -1)) 2>/dev/null
+dd if=/dev/zero of=${IMG_DIR}/home.img bs=512 count=1 seek=$(($HOME_SIZE -1)) 2>/dev/null
+dd if=/dev/zero of=${IMG_DIR}/usr.img bs=512 count=1 seek=$(($USR_SIZE -1)) 2>/dev/null
+
+# Create the empty image where we later will but the partitions in
+dd if=/dev/zero of=${IMG} bs=512 count=1 seek=$(($IMG_SIZE -1))
+
+#
+# Do some math to determine the start addresses of the partitions.
+# Ensure the start of the partitions are always aligned, the end will 
+# always be as we assume the sizes are multiples of 4096 bytes, which
+# is always true as soon as you have an integer multiple of 1MB.
+#
+ROOT_START=8
+HOME_START=$(($ROOT_START + $ROOT_SIZE))
+USR_START=$(($HOME_START + $HOME_SIZE))
+
+set -x
+${CROSS_TOOLS}/nbpartition -m ${IMG} ${ROOT_START} 81:${ROOT_SIZE} 81:${HOME_SIZE} 81:${USR_SIZE}
+set +x
+
+# make the different file system. this part is *also* hacky. We first convert
+# the METALOG.sanitised using mtree into a input METALOG containing uids and
+# gids.
+# Afther that we do some magic processing to add device nodes (also missing from METALOG)
+# and convert the METALOG into a proto file that can be used by mkfs.mfs
+#
+echo "creating the file systems"
+
+#
+# read METALOG and use mtree to conver the user and group names into uid and gids
+# FIX put "input somwhere clean"
+#
+cat ${DESTDIR}/METALOG.sanitised | ${CROSS_TOOLS}/nbmtree -N ${DESTDIR}/etc -C > ${IMG_DIR}/input
+
+# add fstab
+echo "./etc/fstab type=file uid=0 gid=0 mode=0755 size=747 time=1365060731.000000000" >> ${IMG_DIR}/input
+
+# fill root.img (skipping /usr entries while keeping the /usr directory)
+cat ${IMG_DIR}/input  | grep -v "^./usr/" | ${CROSS_TOOLS}/nbtoproto -b ${DESTDIR} -o ${IMG_DIR}/root.in
+
+#
+# add device nodes somewhere in the middle of the proto file. Better would be to add the entries in the
+# original METALOG
+# grab the first part
+grep -B 10000 "^ dev"  ${IMG_DIR}/root.in >  ${IMG_DIR}/root.proto
+# add the device nodes from the ramdisk
+cat  ${OBJ}/drivers/ramdisk/proto.dev >> ${IMG_DIR}/root.proto
+# and add the rest of the file
+grep -A 10000 "^ dev"  ${IMG_DIR}/root.in | tail -n +2    >>  ${IMG_DIR}/root.proto
+rm ${IMG_DIR}/root.in
+
+#
+# Create proto files for /usr and /home using toproto.
+#
+cat ${IMG_DIR}/input  | grep  "^\./usr/\|^. "  | sed "s,\./usr,\.,g" | ${CROSS_TOOLS}/nbtoproto -b ${DESTDIR}/usr -o ${IMG_DIR}/usr.proto
+cat ${IMG_DIR}/input  | grep  "^\./home/\|^. "  | sed "s,\./home,\.,g" | ${CROSS_TOOLS}/nbtoproto -b ${DESTDIR}/home -o ${IMG_DIR}/home.proto
+
+#
+# Generate /root, /usr and /home partition images.
+#
+echo "Writing Minix filesystem images"
+echo " - ROOT"
+${CROSS_TOOLS}/nbmkfs.mfs -b $((${ROOT_SIZE} / 8)) ${IMG_DIR}/root.img ${IMG_DIR}/root.proto
+echo " - USR"
+${CROSS_TOOLS}/nbmkfs.mfs -b $((${USR_SIZE} / 8))  ${IMG_DIR}/usr.img  ${IMG_DIR}/usr.proto
+echo " - HOME"
+${CROSS_TOOLS}/nbmkfs.mfs -b $((${HOME_SIZE} / 8)) ${IMG_DIR}/home.img ${IMG_DIR}/home.proto
+
+#
+# Merge the partitions into a single image.
+#
+echo "Merging file systems"
+dd if=${IMG_DIR}/root.img of=${IMG} seek=$ROOT_START conv=notrunc
+dd if=${IMG_DIR}/home.img of=${IMG} seek=$HOME_START conv=notrunc
+dd if=${IMG_DIR}/usr.img of=${IMG} seek=$USR_START conv=notrunc
+
+moddir=${DESTDIR}/boot/minix/.temp/
+mods="`( cd $moddir; echo mod* | tr ' ' ',' )`"
+echo "To boot this image on kvm:"
+echo "cd $moddir && kvm -serial stdio -kernel kernel -append \"console=tty00 rootdevname=c0d0p0\" -initrd \"$mods\" -hda `pwd`/$IMG"

tools/Makefile

@@ -76,7 +76,8 @@ SUBDIR=	host-mkdep .WAIT compat .WAIT \
 		 cat cksum \
 		file \
 		.WAIT \
-		pwd_mkdb stat zic
+		pwd_mkdb stat zic \
+		partition
 
 .if ${MKLLVM} != "no"
 # .WAIT between llvm-tblgen and llvm-clang-tblgen ensures install

tools/partition/Makefile

@@ -0,0 +1,4 @@
+HOSTPROGNAME=	${_TOOL_PREFIX}partition
+HOST_SRCDIR=	commands/partition
+
+.include "${.CURDIR}/../Makefile.host"