Assorted fixes

Support "--build"ing arrays with bitmaps.
hot-removal of bitmaps
--re-add of drives recently removed.
assorted extra tests

Signed-off-by: Neil Brown <neilb@cse.unsw.edu.au>

16 files changed, 484 insertions, 61 deletions

diff --git a/Build.c b/Build.c
index 7eb1eed..689aad0 100644
--- a/Build.c
+++ b/Build.c
@@ -57,7 +57,7 @@ int Build(char *mddev, int mdfd, int chunk, int level, int layout,
 	int subdevs = 0, missing_disks = 0;
 	mddev_dev_t dv;
 	int bitmap_fd;
-/*	unsigned long long size = ~0ULL; / * needed for bitmap only */
+	unsigned long long size = ~0ULL;
 
 	/* scan all devices, make sure they really are block devices */
 	for (dv = devlist; dv; dv=dv->next) {
@@ -147,6 +147,8 @@ int Build(char *mddev, int mdfd, int chunk, int level, int layout,
 	}
 	/* now add the devices */
 	for ((i=0), (dv = devlist) ; dv ; i++, dv=dv->next) {
+		unsigned long dsize;
+		int fd;
 		if (strcmp("missing", dv->devname) == 0)
 			continue;
 		if (stat(dv->devname, &stb)) {
@@ -159,6 +161,19 @@ int Build(char *mddev, int mdfd, int chunk, int level, int layout,
 				dv->devname);
 			goto abort;
 		}
+		fd = open(dv->devname, O_RDONLY|O_EXCL);
+		if (fd < 0) {
+			fprintf(stderr, Name ": Cannot open %s: %s\n", 
+				dv->devname, strerror(errno));
+			goto abort;
+		}
+		if (ioctl(fd, BLKGETSIZE, &dsize) == 0 && dsize > 0) {
+			unsigned long long ldsize = dsize;
+			ldsize <<= 9;
+			if (size== 0 || ldsize < size)
+				size = ldsize;
+		}
+		close(fd);
 		if (vers>= 9000) {
 			mdu_disk_info_t disk;
 			disk.number = i;
@@ -193,7 +208,7 @@ int Build(char *mddev, int mdfd, int chunk, int level, int layout,
 					return 1;
 				}
 				if (CreateBitmap(bitmap_file, 1, NULL, bitmap_chunk,
-						 delay, write_behind, 0/* FIXME size */)) {
+						 delay, write_behind, size>>9)) {
 					return 1;
 				}
 				bitmap_fd = open(bitmap_file, O_RDWR);
diff --git a/Grow.c b/Grow.c
index ce536d5..6157261 100644
--- a/Grow.c
+++ b/Grow.c
@@ -215,6 +215,14 @@ int Grow_addbitmap(char *devname, int fd, char *file, int chunk, int delay, int
 		return 1;
 	}
 	if (bmf.pathname[0]) {
+		if (strcmp(file,"none")==0) {
+			if (ioctl(fd, SET_BITMAP_FILE, -1)!= 0) {
+				fprintf(stderr, Name ": failed to remove bitmap %s\n",
+					bmf.pathname);
+				return 1;
+			}
+			return 0;
+		}
 		fprintf(stderr, Name ": %s already has a bitmap (%s)\n",
 			devname, bmf.pathname);
 		return 1;
@@ -224,6 +232,14 @@ int Grow_addbitmap(char *devname, int fd, char *file, int chunk, int delay, int
 		return 1;
 	}
 	if (array.state & (1<<MD_SB_BITMAP_PRESENT)) {
+		if (strcmp(file, "none")==0) {
+			array.state &= ~(1<<MD_SB_BITMAP_PRESENT);
+			if (ioctl(fd, SET_ARRAY_INFO, &array)!= 0) {
+				fprintf(stderr, Name ": failed to remove internal bitmap.\n");
+				return 1;
+			}
+			return 0;
+		}
 		fprintf(stderr, Name ": Internal bitmap already present on %s\n",
 			devname);
 		return 1;
@@ -234,7 +250,10 @@ int Grow_addbitmap(char *devname, int fd, char *file, int chunk, int delay, int
 			array.major_version, array.minor_version);
 		return 1;
 	}
-	if (strcmp(file, "internal") == 0) {
+	if (strcmp(file, "none") == 0) {
+		fprintf(stderr, Name ": no bitmap found on %s\n", devname);
+		return 1;
+	} else if (strcmp(file, "internal") == 0) {
 		int d;
 		for (d=0; d< st->max_devs; d++) {
 			mdu_disk_info_t disk;
@@ -267,8 +286,57 @@ int Grow_addbitmap(char *devname, int fd, char *file, int chunk, int delay, int
 			fprintf(stderr, Name ": failed to set internal bitmap.\n");
 			return 1;
 		}
-	} else
-		abort(); /* FIXME */
+	} else {
+		int uuid[4];
+		int bitmap_fd;
+		int d;
+		int max_devs = st->max_devs;
+		void *super = NULL;
+		if (chunk == UnSet)
+			chunk = DEFAULT_BITMAP_CHUNK;
+
+		/* try to load a superblock */
+		for (d=0; d<max_devs; d++) {
+			mdu_disk_info_t disk;
+			char *dv;
+			int fd2;
+			disk.number = d;
+			if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
+				continue;
+			if ((disk.major==0 && disk.minor==0) ||
+			    (disk.state & (1<<MD_DISK_REMOVED)))
+				continue;
+			dv = map_dev(disk.major, disk.minor);
+			if (!dv) continue;
+			fd2 = open(dv, O_RDONLY);
+			if (fd2 >= 0 &&
+			    st->ss->load_super(st, fd2, &super, NULL) == 0) {
+				close(fd2);
+				st->ss->uuid_from_super(uuid, super);
+				break;
+			}
+			close(fd2);
+		}
+		if (d == max_devs) {
+			fprintf(stderr, Name ": cannot find UUID for array!\n");
+			return 1;
+		}
+		if (CreateBitmap(file, 0, (char*)uuid, chunk,
+				 delay, write_behind, array.size*2ULL)) {
+			return 1;
+		}
+		bitmap_fd = open(file, O_RDWR);
+		if (bitmap_fd < 0) {
+			fprintf(stderr, Name ": weird: %s cannot be openned\n",
+				file);
+			return 1;
+		}
+		if (ioctl(fd, SET_BITMAP_FILE, bitmap_fd) < 0) {
+			fprintf(stderr, Name ": Cannot set bitmap file for %s: %s\n",
+				devname, strerror(errno));
+			return 1;
+		}
+	}
 
 	return 0;
 }
diff --git a/Manage.c b/Manage.c
index 8c2da3b..4cf40e1 100644
--- a/Manage.c
+++ b/Manage.c
@@ -172,6 +172,9 @@ int Manage_subdevs(char *devname, int fd,
 	int tfd;
 	struct supertype *st;
 	void *dsuper = NULL;
+	void *osuper = NULL; /* original super */
+	int duuid[4];
+	int ouuid[4];
 
 	if (ioctl(fd, GET_ARRAY_INFO, &array)) {
 		fprintf(stderr, Name ": cannot get array info for %s\n",
@@ -196,6 +199,14 @@ int Manage_subdevs(char *devname, int fd,
 			return 1;
 		case 'a':
 			/* add the device - hot or cold */
+			st = super_by_version(array.major_version,
+					      array.minor_version);
+			if (!st) {
+				fprintf(stderr, Name ": unsupport array - version %d.%d\n",
+					array.major_version, array.minor_version);
+				return 1;
+			}
+
 			/* Make sure it isn't in use (in 2.6 or later) */
 			tfd = open(dv->devname, O_RDONLY|O_EXCL);
 			if (tfd < 0) {
@@ -203,7 +214,11 @@ int Manage_subdevs(char *devname, int fd,
 					dv->devname, strerror(errno));
 				return 1;
 			}
+			if (array.not_persistent==0)
+				st->ss->load_super(st, tfd, &osuper, NULL);
+			/* will use osuper later */
 			close(tfd);
+
 			if (array.major_version == 0 &&
 			    md_get_version(fd)%100 < 2) {
 				if (ioctl(fd, HOT_ADD_DISK,
@@ -219,40 +234,65 @@ int Manage_subdevs(char *devname, int fd,
 				return 1;
 			}
 
-			/* need to find a sample superblock to copy, and
-			 * a spare slot to use 
-			 */
-			st = super_by_version(array.major_version,
-					      array.minor_version);
-			if (!st) {
-				fprintf(stderr, Name ": unsupport array - version %d.%d\n",
-					array.major_version, array.minor_version);
-				return 1;
-			}
-			for (j=0; j<st->max_devs; j++) {
-				char *dev;
-				int dfd;
-				disc.number = j;
-				if (ioctl(fd, GET_DISK_INFO, &disc))
-					continue;
-				if (disc.major==0 && disc.minor==0)
-					continue;
-				if ((disc.state & 4)==0) continue; /* sync */
-				/* Looks like a good device to try */
-				dev = map_dev(disc.major, disc.minor);
-				if (!dev) continue;
-				dfd = open(dev, O_RDONLY);
-				if (dfd < 0) continue;
-				if (st->ss->load_super(st, dfd, &dsuper, NULL)) {
+			if (array.not_persistent == 0) {
+
+				/* need to find a sample superblock to copy, and
+				 * a spare slot to use 
+				 */
+				for (j=0; j<st->max_devs; j++) {
+					char *dev;
+					int dfd;
+					disc.number = j;
+					if (ioctl(fd, GET_DISK_INFO, &disc))
+						continue;
+					if (disc.major==0 && disc.minor==0)
+						continue;
+					if ((disc.state & 4)==0) continue; /* sync */
+					/* Looks like a good device to try */
+					dev = map_dev(disc.major, disc.minor);
+					if (!dev) continue;
+					dfd = open(dev, O_RDONLY);
+					if (dfd < 0) continue;
+					if (st->ss->load_super(st, dfd, &dsuper, NULL)) {
+						close(dfd);
+						continue;
+					}
 					close(dfd);
-					continue;
+					break;
+				}
+				if (!dsuper) {
+					fprintf(stderr, Name ": cannot find valid superblock in this array - HELP\n");
+					return 1;
+				}
+				/* Possibly this device was recently part of the array
+				 * and was temporarily removed, and is now being re-added.
+				 * If so, we can simply re-add it.
+				 */
+				st->ss->uuid_from_super(duuid, dsuper);
+			
+				if (osuper) {
+					st->ss->uuid_from_super(ouuid, osuper);
+					if (memcmp(duuid, ouuid, sizeof(ouuid))==0) {
+						/* look close enough for now.  Kernel
+						 * will worry about where a bitmap
+						 * based reconstruct is possible
+						 */
+						struct mdinfo mdi;
+						struct mddev_ident_s ident;
+						st->ss->getinfo_super(&mdi, &ident, osuper);
+						disc.major = major(stb.st_rdev);
+						disc.minor = minor(stb.st_rdev);
+						disc.number = mdi.disk.number;
+						disc.raid_disk = mdi.disk.raid_disk;
+						disc.state = mdi.disk.state;
+						if (ioctl(fd, ADD_NEW_DISK, &disc) == 0) {
+							if (verbose >= 0)
+								fprintf(stderr, Name ": re-added %s\n", dv->devname);
+							return 0;
+						}
+						/* fall back on normal-add */
+					}
 				}
-				close(dfd);
-				break;
-			}
-			if (!dsuper) {
-				fprintf(stderr, Name ": cannot find valid superblock in this array - HELP\n");
-				return 1;
 			}
 			for (j=0; j< st->max_devs; j++) {
 				disc.number = j;
@@ -267,11 +307,41 @@ int Manage_subdevs(char *devname, int fd,
 			disc.minor = minor(stb.st_rdev);
 			disc.number =j;
 			disc.state = 0;
-			if (dv->writemostly)
-				disc.state |= 1 << MD_DISK_WRITEMOSTLY;
-			st->ss->add_to_super(dsuper, &disc);
-			if (st->ss->write_init_super(st, dsuper, &disc, dv->devname))
-				return 1;
+			if (array.not_persistent==0) {
+				if (dv->writemostly)
+					disc.state |= 1 << MD_DISK_WRITEMOSTLY;
+				st->ss->add_to_super(dsuper, &disc);
+				if (st->ss->write_init_super(st, dsuper, &disc, dv->devname))
+					return 1;
+			} else if (dv->re_add) {
+				/*  this had better be raid1.
+				 * As we are "--re-add"ing we must find a spare slot
+				 * to fill.
+				 */
+				char *used = malloc(array.raid_disks);
+				memset(used, 0, array.raid_disks);
+				for (j=0; j< st->max_devs; j++) {
+					mdu_disk_info_t disc2;
+					disc2.number = j;
+					if (ioctl(fd, GET_DISK_INFO, &disc2))
+						continue;
+					if (disc2.major==0 && disc2.minor==0)
+						continue;
+					if (disc2.state & 8) /* removed */
+						continue;
+					if (disc2.raid_disk < 0)
+						continue;
+					if (disc2.raid_disk > array.raid_disks)
+						continue;
+					used[disc2.raid_disk] = 1;
+				}
+				for (j=0 ; j<array.raid_disks; j++)
+					if (!used[j]) {
+						disc.raid_disk = j;
+						disc.state |= (1<<MD_DISK_SYNC);
+						break;
+					}
+			}
 			if (ioctl(fd,ADD_NEW_DISK, &disc)) {
 				fprintf(stderr, Name ": add new device failed for %s as %d: %s\n",
 					dv->devname, j, strerror(errno));
diff --git a/ReadMe.c b/ReadMe.c
index 2a41854..5dd9874 100644
--- a/ReadMe.c
+++ b/ReadMe.c
@@ -134,6 +134,7 @@ struct option long_options[] = {
     {"bitmap-chunk", 1, 0, 4},
     {"write-behind", 2, 0, 5},
     {"write-mostly",0, 0, 'W'},
+    {"re-add",    0, 0,  6},
 
     /* For assemble */
     {"uuid",      1, 0, 'u'},
diff --git a/mdadm.8 b/mdadm.8
index 9f0dd15..d1e1b8e 100644
--- a/mdadm.8
+++ b/mdadm.8
@@ -225,8 +225,18 @@ slow link.
 .TP
 .BR -b ", " --bitmap=
 Give the name of a bitmap file to use with this array.  Can be used
-with --create (file should not exist) or --assemble (file should
-exist).
+with --create (file should not exist), --assemble (file should
+exist), of --grow (file should not exist).
+
+The file
+.B internal
+can be used to indicate that the bitmap should be stored in the array,
+near the superblock.  There is a limited amount of space for such
+bitmaps, but it is often sufficient.
+
+The file
+.B none
+can be given when used with --grow to remove a bitmap.
 
 .TP
 .BR --bitmap-chunk=
@@ -621,6 +631,20 @@ counts of total, working, active, failed, and spare devices.
 hotadd listed devices.
 
 .TP
+.BR --re-add
+Listed devices are assumed to have recently been part of the array,
+and they are re-added.  This is only different from --add when a
+write-intent bitmap is present.  It causes only those parts of the
+device that have changed since the device was removed from the array
+to be reconstructed.
+
+This flag is only needed with arrays that are built without a
+superblock (i.e. --build, not --create).  For array with a superblock,
+.I mdadm
+checks if a superblock is present and automatically determines if a
+re-add is appropriate. 
+
+.TP
 .BR -r ", " --remove
 remove listed devices.  They must not be active.  i.e. they should
 be failed or spare devices.
@@ -1268,7 +1292,7 @@ change the "raid-disks" attribute of RAID1.
 add a write-intent bitmap to a RAID1 array.
 .PP
 
-Normally when an array is build the "size" it taken from the smallest
+Normally when an array is built the "size" it taken from the smallest
 of the drives.  If all the small drives in an arrays are, one at a
 time, removed and replaced with larger drives, then you could have an
 array of large drives with only a small amount used.  In this
@@ -1294,6 +1318,12 @@ devices that which were in those slots must be failed and removed.
 When the number of devices is increased, any hot spares that are
 present may be activated immediately.
 
+A write-intent bitmap can be added to, or remove from, an active RAID1
+array.  Either internal bitmap, of bitmaps stored in a separate file
+can be added.  Note that if you add a bitmap stored in a file which is
+in a filesystem that is on the raid array being affected, the system
+will deadlock.  The bitmap must be on a separate filesystem.
+
 .SH EXAMPLES
 
 .B "  mdadm --query /dev/name-of-device"
diff --git a/mdadm.c b/mdadm.c
index a24778e..bd29882 100644
--- a/mdadm.c
+++ b/mdadm.c
@@ -92,6 +92,7 @@ int main(int argc, char *argv[])
 	int oneshot = 0;
 	struct supertype *ss = NULL;
 	int writemostly = 0;
+	int re_add = 0;
 
 	int copies;
 
@@ -167,6 +168,7 @@ int main(int argc, char *argv[])
 		case 'a':
 		case 'r':
 		case 'f':
+		case 6: /* re-add */
 			if (!mode) newmode = MANAGE; 
 			break;
 
@@ -222,6 +224,7 @@ int main(int argc, char *argv[])
 					dv->devname = optarg;
 					dv->disposition = devmode;
 					dv->writemostly = writemostly;
+					dv->re_add = re_add;
 					dv->next = NULL;
 					*devlistend = dv;
 					devlistend = &dv->next;
@@ -271,6 +274,7 @@ int main(int argc, char *argv[])
 			dv->devname = optarg;
 			dv->disposition = devmode;
 			dv->writemostly = writemostly;
+			dv->re_add = re_add;
 			dv->next = NULL;
 			*devlistend = dv;
 			devlistend = &dv->next;
@@ -667,6 +671,11 @@ int main(int argc, char *argv[])
 		case O(GROW,'a'):
 		case O(MANAGE,'a'): /* add a drive */
 			devmode = 'a';
+			re_add = 0;
+			continue;
+		case O(MANAGE,6):
+			devmode = 'a';
+			re_add = 1;
 			continue;
 		case O(MANAGE,'r'): /* remove a drive */
 			devmode = 'r';
@@ -960,16 +969,6 @@ int main(int argc, char *argv[])
 				rv |= 1;
 				break;
 			}
-			bitmap_fd = open(bitmap_file, O_RDWR,0);
-			if (bitmap_fd < 0 && errno != ENOENT) {
-				perror(Name ": cannot create bitmap file");
-				rv |= 1;
-				break;
-			}
-			if (bitmap_fd < 0) {
-				bitmap_fd = CreateBitmap(bitmap_file, force, NULL,
-							 bitmap_chunk, delay, write_behind, size);
-			}
 		}
 		rv = Build(devlist->devname, mdfd, chunk, level, layout,
 			   raiddisks, devlist->next, assume_clean,
diff --git a/mdadm.h b/mdadm.h
index 015d470..547519c 100644
--- a/mdadm.h
+++ b/mdadm.h
@@ -137,6 +137,7 @@ typedef struct mddev_dev_s {
 				 * Not set for names read from .config
 				 */
 	char writemostly;
+	char re_add;
 	struct mddev_dev_s *next;
 } *mddev_dev_t;
 
diff --git a/test b/test
index 5008c49..1e7e841 100644
--- a/test
+++ b/test
@@ -19,8 +19,6 @@ then
    echo >&2 "test: $mdadm isn't usable."
 fi
 
-export check="sh $dir/tests/check"
-
 # assume md0, md1, md2 exist in /dev
 md0=/dev/md0 md1=/dev/md1 md2=/dev/md2
 
@@ -46,6 +44,7 @@ do
    [ -f $targetdir/mdtest$d ] || dd if=/dev/zero of=$targetdir/mdtest$d count=$size bs=1K > /dev/null 2>&1
    losetup /dev/loop$d $targetdir/mdtest$d
    eval dev$d=/dev/loop$d
+   eval file$d=$targetdir/mdtest$d
    eval devlist=\"\$devlist \$dev$d\"
 done
 
@@ -86,6 +85,17 @@ check() {
    		echo >&2 "ERROR state $2 not found!"; cat /proc/mdstat ; exit 1; }
        sleep 0.5
       ;;
+
+    bitmap )
+       grep -s bitmap > /dev/null /proc/mdstat || {
+	   echo >&2 ERROR no bitmap ; cat /proc/mdstat ; exist 1; }
+      ;;
+    nobitmap )
+       if grep -s "bitmap" > /dev/null /proc/mdstat 
+       then
+	   echo >&2 ERROR bitmap present ; cat /proc/mdstat ; exit 1;
+       fi
+      ;;
    
     * ) echo >&2 ERROR unknown check $1 ; exit 1;
    esac
diff --git a/tests/03r5assemV1 b/tests/03r5assemV1
new file mode 100644
index 0000000..7c72ad6
--- /dev/null
+++ b/tests/03r5assemV1
@@ -0,0 +1,109 @@
+
+# create a v-1 raid5 array and assemble in various ways
+
+mdadm -CR -e1 --name one $md1 -l5 -n3 $dev0 $dev1 $dev2 
+tst="check raid5 ;testdev $md1 2 $mdsize1 64 ; mdadm -S $md1"
+uuid=`mdadm -Db $md1 | sed 's/.*UUID=//'`
+check wait
+
+eval $tst
+
+mdadm -A $md1 $dev0 $dev1 $dev2 
+eval $tst
+
+mdadm -A $md1 -u $uuid $devlist 
+eval $tst
+
+mdadm -A $md1 --name one $devlist  
+eval $tst
+
+
+conf=$targetdir/mdadm.conf
+{ 
+  echo DEVICE $devlist
+  echo array $md1 UUID=$uuid
+} > $conf
+
+mdadm -As -c $conf $md1 
+eval $tst
+
+{ 
+  echo DEVICE $devlist
+  echo array $md1 name=one
+} > $conf
+
+mdadm -As -c $conf  
+eval $tst
+
+{ 
+  echo DEVICE $devlist
+  echo array $md1 devices=$dev0,$dev1,$dev2
+} > $conf
+
+mdadm -As -c $conf 
+
+echo "DEVICE $devlist" > $conf
+mdadm -Db $md1 >> $conf
+eval $tst
+
+mdadm --assemble --scan --config=$conf $md1 
+eval $tst
+
+echo "  metadata=1.0 devices=$dev0,$dev1,$dev2" >> $conf
+mdadm --assemble --scan --config=$conf $md1 
+eval $tst
+
+### Now with a missing device
+
+mdadm -AR $md1 $dev0 $dev2 # 
+check state U_U
+eval $tst
+
+mdadm -A $md1 -u $uuid $devlist 
+check state U_U
+eval $tst
+
+mdadm -A $md1 --name=one $devlist  
+check state U_U
+eval $tst
+
+
+conf=$targetdir/mdadm.conf
+{ 
+  echo DEVICE $devlist
+  echo array $md1 UUID=$uuid
+} > $conf
+
+mdadm -As -c $conf $md1 
+check state U_U
+eval $tst
+
+{ 
+  echo DEVICE $devlist
+  echo array $md1 name=one
+} > $conf
+
+mdadm -As -c $conf  
+check state U_U
+eval $tst
+
+{ 
+  echo DEVICE $devlist
+  echo array $md1 devices=$dev0,$dev1,$dev2
+} > $conf
+
+mdadm -As -c $conf 
+
+echo "DEVICE $devlist" > $conf
+mdadm -Db $md1 >> $conf
+check state U_U
+eval $tst
+
+mdadm --assemble --scan --config=$conf $md1 
+check state U_U
+eval $tst
+
+echo "  metadata=1.0 devices=$dev0,$dev1,$dev2" >> $conf
+mdadm --assemble --scan --config=$conf $md1 
+check state U_U
+eval $tst
diff --git a/tests/05r1-grow-external b/tests/05r1-grow-external
index ba9b3e5..73fccd9 100644
--- a/tests/05r1-grow-external
+++ b/tests/05r1-grow-external
@@ -8,7 +8,7 @@ testdev $md0 1 $mdsize0 1
 
 bmf=$targetdir/bm
 rm -f $bmf
-mdadm -E $dev1
+#mdadm -E $dev1
 mdadm --grow $md0 --bitmap=$bmf --delay=1 || { $mdadm -X $bmf ; exit 1; }
 dirty1=`mdadm -X $bmf | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
 sleep 4
@@ -19,10 +19,15 @@ dirty3=`mdadm -X $bmf | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
 sleep 4
 dirty4=`mdadm -X $bmf | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
 
-echo $dirty1 $dirty2 $dirty3 $dirty4
+#echo $dirty1 $dirty2 $dirty3 $dirty4
 if [ $dirty2 -ne 0 -o $dirty4 -ne 0 -o $dirty3 -lt 400 ]
 then 
    echo bad dirty counts
    exit 1
 fi
+
+# now to remove the bitmap
+check bitmap
+mdadm --grow $md0 --bitmap=none
+check nobitmap
 mdadm -S $md0
diff --git a/tests/05r1-grow-internal b/tests/05r1-grow-internal
new file mode 100644
index 0000000..addae35
--- /dev/null
+++ b/tests/05r1-grow-internal
@@ -0,0 +1,31 @@
+
+#
+# create a raid1 array, add an internal bitmap
+#
+mdadm --create --run $md0 -l 1 -n 2 $dev1 $dev2
+check wait
+testdev $md0 1 $mdsize0 1
+
+#mdadm -E $dev1
+mdadm --grow $md0 --bitmap=internal --delay=1 || $mdadm -X $dev2
+dirty1=`mdadm -X $dev2 | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
+sleep 4
+dirty2=`mdadm -X $dev2 | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
+
+testdev $md0 1 $mdsize0 1
+dirty3=`mdadm -X $dev2 | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
+sleep 4
+dirty4=`mdadm -X $dev2 | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
+
+#echo $dirty1 $dirty2 $dirty3 $dirty4
+if [ $dirty2 -ne 0 -o $dirty4 -ne 0 -o $dirty3 -lt 400 ]
+then 
+   echo bad dirty counts
+   exit 1
+fi
+
+# now to remove the bitmap
+check bitmap
+mdadm --grow $md0 --bitmap=none
+check nobitmap
+mdadm -S $md0
diff --git a/tests/05r1-re-add b/tests/05r1-re-add
new file mode 100644
index 0000000..733f8e7
--- /dev/null
+++ b/tests/05r1-re-add
@@ -0,0 +1,36 @@
+
+#
+# create a raid1, remove a drive, and readd it.
+# resync should be instant.
+# Then do some IO first.  Resync should still be very fast
+#
+
+mdadm -CR $md0 -l1 -n2 -binternal -d1 $dev1 $dev2
+check resync
+check wait
+testdev $md0 1 $mdsize0 1
+sleep 4
+
+mdadm $md0 -f $dev2
+sleep 1
+mdadm $md0 -r $dev2
+mdadm $md0 -a $dev2
+ cat /proc/mdstat
+check nosync
+
+mdadm $md0 -f $dev2
+sleep 1
+mdadm $md0 -r $dev2
+testdev $md0 1 $mdsize0 1
+mdadm $md0 -a $dev2
+check wait
+cmp --bytes=$[$mdsize0*1024] $dev1 $dev2
+
+mdadm $md0 -f $dev2; sleep 1
+mdadm $md0 -r $dev2
+if dd if=/dev/zero of=$md0 ; then : ; fi
+mdadm $md0 -a $dev2
+check recovery
+check wait
+cmp --bytes=$[$mdsize0*1024] $dev1 $dev2
+mdadm -S $md0
diff --git a/tests/05r1-re-add-nosuper b/tests/05r1-re-add-nosuper
new file mode 100644
index 0000000..8025a68
--- /dev/null
+++ b/tests/05r1-re-add-nosuper
@@ -0,0 +1,37 @@
+
+#
+# create a raid1, remove a drive, and readd it.
+# resync should be instant.
+# Then do some IO first.  Resync should still be very fast
+#
+bmf=$targetdir/bitmap2
+rm -f $bmf
+mdadm -B $md0 -l1 -n2 -b$bmf -d1 $dev1 $dev2
+check resync
+check wait
+testdev $md0 1 $size 1
+sleep 4
+
+mdadm $md0 -f $dev2
+sleep 1
+mdadm $md0 -r $dev2
+mdadm $md0 --re-add $dev2
+check nosync
+
+mdadm $md0 -f $dev2
+sleep 1
+mdadm $md0 -r $dev2
+testdev $md0 1 $size 1
+mdadm $md0 --re-add $dev2
+check wait
+cmp --bytes=$[$mdsize0*1024] $dev1 $dev2
+
+mdadm $md0 -f $dev2; sleep 1
+mdadm $md0 -r $dev2
+if dd if=/dev/zero of=$md0 ; then : ; fi
+mdadm $md0 --re-add $dev2
+check recovery
+check wait
+# should BLKFLSBUF and then read $dev1/$dev2...
+cmp --bytes=$[$mdsize0*1024] $file1 $file2
+mdadm -S $md0
diff --git a/tests/06name b/tests/06name
index cbcd094..9c749f2 100644
--- a/tests/06name
+++ b/tests/06name
@@ -8,6 +8,6 @@ mdadm -D $md0 | grep 'Name : Fred$' > /dev/null || exit 1
 mdadm -S $md0
 
 mdadm -A $md0 --name="Fred" $devlist
-mdadm -Db $md0
+#mdadm -Db $md0
 mdadm -S $md0
 
diff --git a/tests/06r5swap b/tests/06r5swap
index 41e8be6..e414f76 100644
--- a/tests/06r5swap
+++ b/tests/06r5swap
@@ -14,5 +14,5 @@ diff -u $targetdir/d1 $targetdir/d1s
 
 mdadm --assemble --update=byteorder $md0 $dev0 $dev1 $dev2 $dev3
 sleep 3
-cat /proc/mdstat
+check recovery
 mdadm -S $md0
diff --git a/tests/06sysfs b/tests/06sysfs
new file mode 100644
index 0000000..af63ef4
--- /dev/null
+++ b/tests/06sysfs
@@ -0,0 +1,11 @@
+exit 0
+mdadm -CR $md0 -l1 -n3 $dev1 $dev2 $dev3
+
+ls -Rl /sys/block/md0
+
+cat /sys/block/md0/md/level
+cat /sys/block/md0/md/raid_disks
+
+mdadm -S $md0
+
+exit 1