From e86c9dd6d847ec57ec400b118efaf2c1808f10bc Mon Sep 17 00:00:00 2001 From: Neil Brown Date: Mon, 13 Mar 2006 05:51:32 +0000 Subject: Initial reshape support Needs work for other levels etc. Signed-off-by: Neil Brown --- Grow.c | 408 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 407 insertions(+), 1 deletion(-) (limited to 'Grow.c') diff --git a/Grow.c b/Grow.c index 9e8e217..ece2bda 100644 --- a/Grow.c +++ b/Grow.c @@ -306,7 +306,7 @@ int Grow_addbitmap(char *devname, int fd, char *file, int chunk, int delay, int continue; if (st->ss->load_super(st, fd2, &super, NULL)==0) { if (st->ss->add_internal_bitmap(st, super, - chunk, delay, write_behind, + chunk, delay, write_behind, bitmapsize, 0, major)) st->ss->write_bitmap(st, fd2, super); else { @@ -378,4 +378,410 @@ int Grow_addbitmap(char *devname, int fd, char *file, int chunk, int delay, int return 0; } + +/* + * When reshaping an array we might need to backup some data. + * This is written to all spares with a 'super_block' describing it. + * The superblock goes 1K form the end of the used space on the + * device. + * It if written after the backup is complete. + * It has the following structure. + */ + +struct mdp_backup_super { + char magic[16]; /* md_backup_data-1 */ + __u8 set_uuid[16]; + __u64 mtime; + /* start/sizes in 512byte sectors */ + __u64 devstart; + __u64 arraystart; + __u64 length; + __u32 sb_csum; /* csum of preceeding bytes. */ +}; + +int bsb_csum(char *buf, int len) +{ + int i; + int csum = 0; + for (i=0; i= 0) { + fprintf(stderr, Name ": %s: Cannot change size of a 'faulty' array\n", + devname); + return 1; + } + if (level != UnSet && level != LEVEL_FAULTY) { + fprintf(stderr, Name ": %s: Cannot change RAID level of a 'faulty' array\n", + devname); + return 1; + } + if (chunksize || raid_disks) { + fprintf(stderr, Name ": %s: Cannot change chunksize or disks of a 'faulty' array\n", + devname); + return 1; + } + if (layout == UnSet) + return 0; /* nothing to do.... */ + + array.layout = layout; + if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) { + fprintf(stderr, Name ": Cannot set layout for %s: %s\n", + devname, strerror(errno)); + return 1; + } + if (!quiet) + printf("layout for %s set to %d\n", devname, array.layout); + return 0; + + case 1: /* raid_disks and size can each be changed. They are independant */ + + if (level != UnSet && level != 1) { + fprintf(stderr, Name ": %s: Cannot change RAID level of a RAID1 array.\n", + devname); + return 1; + } + if (chunksize || layout != UnSet) { + fprintf(stderr, Name ": %s: Cannot change chunk size of layout for a RAID1 array.\n", + devname); + return 1; + } + + /* Each can trigger a resync/recovery which will block the + * other from happening. Later we could block + * resync for the duration via 'sync_action'... + */ + if (raid_disks >= 0) + array.raid_disks = raid_disks; + if (size >= 0) + array.size = size; + if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) { + fprintf(stderr, Name ": Cannot set device size/shape for %s: %s\n", + devname, strerror(errno)); + return 1; + } + return 0; + + case 4: + case 5: + case 6: + st = super_by_version(array.major_version, + array.minor_version); + /* size can be changed independantly. + * layout/chunksize/raid_disks/level can be changed + * though the kernel may not support it all. + * If 'suspend_lo' is not present in devfs, then + * these cannot be changed. + */ + if (size >= 0) { + /* Cannot change other details as well.. */ + if (layout != UnSet || + chunksize != 0 || + raid_disks != 0 || + level != UnSet) { + fprintf(stderr, Name ": %s: Cannot change shape as well as size of a %s array.\n", + devname, c); + return 1; + } + array.size = size; + if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) { + fprintf(stderr, Name ": Cannot set device size/shape for %s: %s\n", + devname, strerror(errno)); + return 1; + } + return 0; + } + /* Ok, just change the shape. This can be awkward. + * There are three possibilities. + * 1/ The array will shrink. We don't support this + * possibility. Maybe one day... + * 2/ The array will not change size. This is easy enough + * to do, but not reliably. If the process is aborted + * the array *will* be corrupted. So maybe we can allow + * this but only if the user is really certain. e.g. + * --really-risk-everything + * 3/ The array will grow. This can be reliably achieved. + * However the kernel's restripe routines will cheerfully + * overwrite some early data before it is safe. So we + * need to make a backup of the early parts of the array + * and be ready to restore it if rebuild aborts very early. + * + * We backup data by writing it to all spares (there must be + * at least 1, so even raid6->raid5 requires a spare to be + * present). + * + * So: we enumerate the devices in the array and + * make sure we can open all of them. + * Then we freeze the early part of the array and + * backup to the various spares. + * Then we request changes and start the reshape. + * Monitor progress until it has passed the danger zone. + * and finally invalidate the copied data and unfreeze the + * start of the array. + * + * Before we can do this we need to decide: + * - will the array grow? Just calculate size + * - how much needs to be saved: count stripes. + * - where to save data... good question. + * + */ + nlevel = olevel = array.level; + nchunk = ochunk = array.chunk_size; + nlayout = olayout = array.layout; + ndisks = odisks = array.raid_disks; + + if (level != UnSet) nlevel = level; + if (chunksize) nchunk = chunksize; + if (layout != UnSet) nlayout = layout; + if (raid_disks) ndisks = raid_disks; + + odata = odisks-1; + if (olevel == 6) odata--; /* number of data disks */ + ndata = ndisks-1; + if (nlevel == 6) ndata--; + + if (ndata < odata) { + fprintf(stderr, Name ": %s: Cannot reduce number of data disks (yet).\n", + devname); + return 1; + } + if (ndata == odata) { + fprintf(stderr, Name ": %s: Cannot reshape array without increasing size (yet).\n", + devname); + return 1; + } + /* Well, it is growing... so how much do we need to backup. + * Need to backup a full number of new-stripes, such that the + * last one does not over-write any place that it would be read + * from + */ + nstripe = ostripe = 0; + while (nstripe+ochunk/512 >= ostripe) { + nstripe += nchunk/512; + last_block = nstripe * ndata; + ostripe = last_block / odata; + } + printf("Need to backup to stripe %llu sectors, %lluK\n", nstripe, last_block/2); + + sra = sysfs_read(fd, 0, + GET_COMPONENT|GET_DEVS|GET_OFFSET|GET_STATE); + if (!sra) { + fprintf(stderr, Name ": %s: Cannot get array details from sysfs\n", + devname); + return 1; + } + + if (last_block >= sra->component_size/2) { + fprintf(stderr, Name ": %s: Something wrong - reshape aborted\n", + devname); + return 1; + } + + nrdisks = array.nr_disks + sra->spares; + /* Now we need to open all these devices so we can read/write. + */ + fdlist = malloc(nrdisks * sizeof(int)); + offsets = malloc(nrdisks * sizeof(offsets[0])); + if (!fdlist || !offsets) { + fprintf(stderr, Name ": malloc failed: grow aborted\n"); + return 1; + } + for (d=0; d< nrdisks; d++) + fdlist[d] = -1; + d = array.raid_disks; + for (sd = sra->devs; sd; sd=sd->next) { + if (sd->state & (1<state & (1<major, sd->minor); + fdlist[sd->role] = open(dn, O_RDONLY); + offsets[sd->role] = sd->offset; + if (fdlist[sd->role] < 0) { + fprintf(stderr, Name ": %s: cannot open component %s\n", + devname, dn); + goto abort; + } + } else { + /* spare */ + char *dn = map_dev(sd->major, sd->minor); + fdlist[d] = open(dn, O_RDWR); + offsets[d] = sd->offset; + if (fdlist[d]<0) { + fprintf(stderr, Name ": %s: cannot open component %s\n", + devname, dn); + goto abort; + } + d++; + } + } + for (i=0 ; iss->load_super(st, fdlist[0], &super, NULL)) { + fprintf(stderr, Name ": %s: Cannot find a superblock\n", + devname); + goto abort; + } + + spares = sra->spares; + + /* Decide offset for the backup and llseek the spares */ + for (i=array.raid_disks; icomponent_size - last_block - 8; + if (lseek64(fdlist[i], offsets[i]<<9, 0) != offsets[i]<<9) { + fprintf(stderr, Name ": could not seek...\n"); + goto abort; + } + } + array.level = nlevel; + array.raid_disks = ndisks; + array.chunk_size = nchunk; + array.layout = nlayout; + if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) { + fprintf(stderr, Name ": Cannot set device size/shape for %s: %s\n", + devname, strerror(errno)); + goto abort; + } + + /* suspend the relevant region */ + sysfs_set_num(sra, NULL, "suspend_hi", 0); /* just in case */ + if (sysfs_set_num(sra, NULL, "suspend_lo", 0) < 0 || + sysfs_set_num(sra, NULL, "suspend_hi", last_block) < 0) { + fprintf(stderr, Name ": %s: failed to suspend device.\n", + devname); + goto abort_resume; + } + + + err = save_stripes(fdlist, offsets, + odisks, ochunk, olevel, olayout, + spares, fdlist+odisks, + 0ULL, nstripe*512); + + /* abort if there was an error */ + if (err < 0) { + fprintf(stderr, Name ": %s: failed to save critical region\n", + devname); + goto abort_resume; + } + /* FIXME write superblocks */ + memcpy(bsb.magic, "md_backups_data-1", 16); + st->ss->uuid_from_super((int*)&bsb.set_uuid, super); + bsb.mtime = time(0); + bsb.arraystart = 0; + bsb.length = last_block; + for (i=odisks; i= nstripe) + break; + sleep(1); + } + /* invalidate superblocks */ + memset(&bsb, 0, sizeof(bsb)); + for (i=odisks; i= 0) + close(fdlist[i]); + free(fdlist); + free(offsets); + + break; + } + return 0; + + + abort_resume: + sysfs_set_num(sra, NULL, "suspend_lo", last_block); + abort: + for (i=0; i= 0) + close(fdlist[i]); + free(fdlist); + free(offsets); + return 1; + +} -- cgit