diff options
| -rw-r--r-- | WHATS_NEW | 1 | ||||
| -rw-r--r-- | daemons/dmeventd/plugins/raid/dmeventd_raid.c | 37 | ||||
| -rw-r--r-- | doc/example.conf.in | 26 | ||||
| -rw-r--r-- | doc/lvm2-raid.txt | 71 | ||||
| -rw-r--r-- | doc/lvm_fault_handling.txt | 101 | ||||
| -rw-r--r-- | lib/config/defaults.h | 1 | ||||
| -rw-r--r-- | tools/lvconvert.c | 54 |
7 files changed, 227 insertions, 64 deletions
@@ -1,5 +1,6 @@ Version 2.02.89 - ================================== + Add policy based automated repair of RAID logical volumes Don't allow two images to be split and tracked from a RAID LV at one time Don't allow size change of RAID LV that is tracking changes for a split image Don't allow size change of RAID sub-LVs independently diff --git a/daemons/dmeventd/plugins/raid/dmeventd_raid.c b/daemons/dmeventd/plugins/raid/dmeventd_raid.c index 71820c29..dd5e7eb0 100644 --- a/daemons/dmeventd/plugins/raid/dmeventd_raid.c +++ b/daemons/dmeventd/plugins/raid/dmeventd_raid.c @@ -24,6 +24,41 @@ /* FIXME Replace most syslogs with log_error() style messages and add complete context. */ /* FIXME Reformat to 80 char lines. */ +/* + * run_repair is a close copy to + * plugins/mirror/dmeventd_mirror.c:_remove_failed_devices() + */ +static int run_repair(const char *device) +{ + int r; +#define CMD_SIZE 256 /* FIXME Use system restriction */ + char cmd_str[CMD_SIZE]; + char *vg = NULL, *lv = NULL, *layer = NULL; + + if (strlen(device) > 200) /* FIXME Use real restriction */ + return -1; + + if (!dm_split_lvm_name(dmeventd_lvm2_pool(), device, &vg, &lv, &layer)) { + syslog(LOG_ERR, "Unable to determine VG name from %s.", + device); + return -1; + } + + /* FIXME Is any sanity-checking required on %s? */ + if (CMD_SIZE <= snprintf(cmd_str, CMD_SIZE, "lvconvert --config devices{ignore_suspended_devices=1} --repair --use-policies %s/%s", vg, lv)) { + /* this error should be caught above, but doesn't hurt to check again */ + syslog(LOG_ERR, "Unable to form LVM command: Device name too long."); + return -1; + } + + r = dmeventd_lvm2_run(cmd_str); + + if (r != ECMD_PROCESSED) + syslog(LOG_INFO, "Repair of RAID LV %s/%s failed.", vg, lv); + + return (r == ECMD_PROCESSED) ? 0 : -1; +} + static int _process_raid_event(char *params, const char *device) { int i, n, failure = 0; @@ -71,7 +106,7 @@ static int _process_raid_event(char *params, const char *device) break; } if (failure) - return 0; /* Don't bother parsing rest of status */ + return run_repair(device); } p = strstr(resync_ratio, "/"); diff --git a/doc/example.conf.in b/doc/example.conf.in index 72005e98..223f273d 100644 --- a/doc/example.conf.in +++ b/doc/example.conf.in @@ -522,9 +522,31 @@ activation { # "auto" - Use default value chosen by kernel. readahead = "auto" + # 'raid_fault_policy' defines how a device failure in a RAID logical + # volume is handled. This includes logical volumes that have the following + # segment types: raid1, raid4, raid5*, and raid6*. + # + # In the event of a failure, the following policies will determine what + # actions are performed during the automated response to failures (when + # dmeventd is monitoring the RAID logical volume) and when 'lvconvert' is + # called manually with the options '--repair' and '--use-policies'. + # + # "warn" - Use the system log to warn the user that a device in the RAID + # logical volume has failed. It is left to the user to run + # 'lvconvert --repair' manually to remove or replace the failed + # device. As long as the number of failed devices does not + # exceed the redundancy of the logical volume (1 device for + # raid4/5, 2 for raid6, etc) the logical volume will remain + # usable. + # + # "allocate" - Attempt to use any extra physical volumes in the volume + # group as spares and replace faulty devices. + # + raid_fault_policy = "warn" + # 'mirror_image_fault_policy' and 'mirror_log_fault_policy' define - # how a device failure affecting a mirror is handled. - # A mirror is composed of mirror images (copies) and a log. + # how a device failure affecting a mirror (of "mirror" segment type) is + # handled. A mirror is composed of mirror images (copies) and a log. # A disk log ensures that a mirror does not need to be re-synced # (all copies made the same) every time a machine reboots or crashes. # diff --git a/doc/lvm2-raid.txt b/doc/lvm2-raid.txt index 4bfc0098..a6f09154 100644 --- a/doc/lvm2-raid.txt +++ b/doc/lvm2-raid.txt @@ -380,7 +380,7 @@ until all possible changes have been made. This reduces the likelyhood of bad intermediate stages being left due to a failure of operation or machine crash. RAID1 '--splitmirrors', '--trackchanges', and '--merge' operations ------------------------------------------------------------------ +------------------------------------------------------------------ This suite of operations is only available to the "raid1" segment type. Splitting an image from a RAID1 array is almost identical to the removal of @@ -404,3 +404,72 @@ set as "hidden" again. Recycling the array (suspend/resume) restores the sub-LV to its position in the array and begins the process of sync'ing the changes that were made since the time it was split from the array. +RAID device replacement with '--replace' +---------------------------------------- +This option is available to all RAID segment types. + +The '--replace' option can be used to remove a particular device from a RAID +logical volume and replace it with a different one in one action (CLI command). +The device device to be removed is specified as the argument to the '--replace' +option. This option can be specified more than once in a single command, +allowing multiple devices to be replaced at the same time - provided the RAID +logical volume has the necessary redundancy to allow the action. The devices +to be used as replacements can also be specified in the command; similar to the +way allocatable devices are specified during an up-convert. + +Example> lvconvert --replace /dev/sdd1 --replace /dev/sde1 vg/lv /dev/sd[bc]1 + +RAID '--repair' +--------------- +This 'lvconvert' option is available to all RAID segment types and is described +under "RAID Fault Handling". + + +RAID Fault Handling +=================== +RAID is not like traditional LVM mirroring (i.e. the "mirror" segment type). +LVM mirroring required failed devices to be removed or the logical volume would +simply hang. RAID arrays can keep on running with failed devices. In fact, for +RAID types other than RAID1 removing a device would mean substituting an error +target or converting to a lower level RAID (e.g. RAID6 -> RAID5, or RAID4/5 to +RAID0). Therefore, rather than removing a failed device unconditionally, the +user has a couple of options to choose from. + +The automated response to a device failure is handled according to the user's +preference defined in lvm.conf:activation.raid_fault_policy. The options are: + # "warn" - Use the system log to warn the user that a device in the RAID + # logical volume has failed. It is left to the user to run + # 'lvconvert --repair' manually to remove or replace the failed + # device. As long as the number of failed devices does not + # exceed the redundancy of the logical volume (1 device for + # raid4/5, 2 for raid6, etc) the logical volume will remain + # usable. + # + # "remove" - NOT CURRENTLY IMPLEMENTED OR DOCUMENTED IN example.conf.in. + # Remove the failed device and reduce the RAID logical volume + # accordingly. If a single device dies in a 3-way mirror, + # remove it and reduce the mirror to 2-way. If a single device + # dies in a RAID 4/5 logical volume, reshape it to a striped + # volume, etc - RAID 6 -> RAID 4/5 -> RAID 0. If devices + # cannot be removed for lack of redundancy, fail. + # THIS OPTION CANNOT YET BE IMPLEMENTED BECAUSE RESHAPE IS NOT + # YET SUPPORTED IN linux/drivers/md/dm-raid.c. The superblock + # does not yet hold enough information to support reshaping. + # + # "allocate" - Attempt to use any extra physical volumes in the volume + # group as spares and replace faulty devices. + +If manual intervention is taken, either in response to the automated solution's +"warn" mode or simply because dmeventd hadn't run, then the user can call +'lvconvert --repair vg/lv' and follow the prompts. They will be prompted +whether or not to replace the device and cause a full recovery of the failed +device. + +If replacement is chosen via the manual method or "allocate" is the policy taken +by the automated response, then 'lvconvert --replace' is the mechanism used to +attempt the replacement of the failed device. + +'vgreduce --removemissing' is ineffectual at repairing RAID logical volumes. It +will remove the failed device, but the RAID logical volume will simply continue +to operate with an <unknown> sub-LV. The user should clear the failed device +with 'lvconvert --repair'. diff --git a/doc/lvm_fault_handling.txt b/doc/lvm_fault_handling.txt index fa30c0c2..53b447ea 100644 --- a/doc/lvm_fault_handling.txt +++ b/doc/lvm_fault_handling.txt @@ -15,6 +15,12 @@ from (e.g. a power failure, intermittent network outage, block relocation, etc). The policies for handling both types of failures is described herein. +Users need to be aware that there are two implementations of RAID1 in LVM. +The first is defined by the "mirror" segment type. The second is defined by +the "raid1" segment type. The characteristics of each of these are defined +in lvm.conf under 'mirror_segtype_default' - the configuration setting used to +identify the default RAID1 implementation used for LVM operations. + Available Operations During a Device Failure -------------------------------------------- When there is a device failure, LVM behaves somewhat differently because @@ -51,30 +57,36 @@ are as follows: a linear, stripe, or snapshot device is located on the failed device the command will not proceed without a '--force' option. The result of using the '--force' option is the entire removal and complete - loss of the non-redundant logical volume. Once this operation is - complete, the volume group will again have a complete and consistent - view of the devices it contains. Thus, all operations will be - permitted - including creation, conversion, and resizing operations. + loss of the non-redundant logical volume. If an image or metadata area + of a RAID logical volume is on the failed device, the sub-LV affected is + replace with an error target device - appearing as <unknown> in 'lvs' + output. RAID logical volumes cannot be completely repaired by vgreduce - + 'lvconvert --repair' (listed below) must be used. Once this operation is + complete on volume groups not containing RAID logical volumes, the volume + group will again have a complete and consistent view of the devices it + contains. Thus, all operations will be permitted - including creation, + conversion, and resizing operations. It is currently the preferred method + to call 'lvconvert --repair' on the individual logical volumes to repair + them followed by 'vgreduce --removemissing' to extract the physical volume's + representation in the volume group. - 'lvconvert --repair <VG/LV>': This action is designed specifically - to operate on mirrored logical volumes. It is used on logical volumes - individually and does not remove the faulty device from the volume - group. If, for example, a failed device happened to contain the - images of four distinct mirrors, it would be necessary to run - 'lvconvert --repair' on each of them. The ultimate result is to leave - the faulty device in the volume group, but have no logical volumes - referencing it. In addition to removing mirror images that reside - on failed devices, 'lvconvert --repair' can also replace the failed - device if there are spare devices available in the volume group. The - user is prompted whether to simply remove the failed portions of the - mirror or to also allocate a replacement, if run from the command-line. - Optionally, the '--use-policies' flag can be specified which will - cause the operation not to prompt the user, but instead respect + to operate on individual logical volumes. If, for example, a failed + device happened to contain the images of four distinct mirrors, it would + be necessary to run 'lvconvert --repair' on each of them. The ultimate + result is to leave the faulty device in the volume group, but have no logical + volumes referencing it. (This allows for 'vgreduce --removemissing' to + removed the physical volumes cleanly.) In addition to removing mirror or + RAID images that reside on failed devices, 'lvconvert --repair' can also + replace the failed device if there are spare devices available in the + volume group. The user is prompted whether to simply remove the failed + portions of the mirror or to also allocate a replacement, if run from the + command-line. Optionally, the '--use-policies' flag can be specified which + will cause the operation not to prompt the user, but instead respect the policies outlined in the LVM configuration file - usually, - /etc/lvm/lvm.conf. Once this operation is complete, mirrored logical - volumes will be consistent and I/O will be allowed to continue. - However, the volume group will still be inconsistent - due to the - refernced-but-missing device/PV - and operations will still be + /etc/lvm/lvm.conf. Once this operation is complete, the logical volumes + will be consistent. However, the volume group will still be inconsistent - + due to the refernced-but-missing device/PV - and operations will still be restricted to the aformentioned actions until either the device is restored or 'vgreduce --removemissing' is run. @@ -98,13 +110,15 @@ following possible exceptions exist: Automated Target Response to Failures: -------------------------------------- -The only LVM target type (i.e. "personality") that has an automated -response to failures is a mirrored logical volume. The other target +The only LVM target types (i.e. "personalities") that have an automated +response to failures are the mirror and RAID logical volumes. The other target types (linear, stripe, snapshot, etc) will simply propagate the failure. [A snapshot becomes invalid if its underlying device fails, but the origin will remain valid - presuming the origin device has not failed.] -There are three types of errors that a mirror can suffer - read, write, -and resynchronization errors. Each is described in depth below. + +Starting with the "mirror" segment type, there are three types of errors that +a mirror can suffer - read, write, and resynchronization errors. Each is +described in depth below. Mirror read failures: If a mirror is 'in-sync' (i.e. all images have been initialized and @@ -184,38 +198,5 @@ command are set in the LVM configuration file. They are: choice of when to incure the extra performance costs of replacing the failed image. -TODO... -The appropriate time to take permanent corrective action on a mirror -should be driven by policy. There should be a directive that takes -a time or percentage argument. Something like the following: -- mirror_fault_policy_WHEN = "10sec"/"10%" -A time value would signal the amount of time to wait for transient -failures to resolve themselves. The percentage value would signal the -amount a mirror could become out-of-sync before the faulty device is -removed. - -A mirror cannot be used unless /some/ corrective action is taken, -however. One option is to replace the failed mirror image with an -error target, forgo the use of 'handle_errors', and simply let the -out-of-sync regions accumulate and be tracked by the log. Mirrors -that have more than 2 images would have to "stack" to perform the -tracking, as each failed image would have to be associated with a -log. If the failure is transient, the device would replace the -error target that was holding its spot and the log that was tracking -the deltas would be used to quickly restore the portions that changed. - -One unresolved issue with the above scheme is how to know which -regions of the mirror are out-of-sync when a problem occurs. When -a write failure occurs in the kernel, the log will contain those -regions that are not in-sync. If the log is a disk log, that log -could continue to be used to track differences. However, if the -log was a core log - or if the log device failed at the same time -as an image device - there would be no way to determine which -regions are out-of-sync to begin with as we start to track the -deltas for the failed image. I don't have a solution for this -problem other than to only be able to handle errors in this way -if conditions are right. These issues will have to be ironed out -before proceeding. This could be another case, where it is better -to handle failures in the kernel by allowing the kernel to store -updates in various metadata areas. -...TODO +RAID logical volume device failures are handled differently from the "mirror" +segment type. Discussion of this can be found in lvm2-raid.txt. diff --git a/lib/config/defaults.h b/lib/config/defaults.h index 27acf8ab..a86016ea 100644 --- a/lib/config/defaults.h +++ b/lib/config/defaults.h @@ -55,6 +55,7 @@ #define DEFAULT_MIRROR_LOG_FAULT_POLICY "allocate" #define DEFAULT_MIRROR_IMAGE_FAULT_POLICY "remove" #define DEFAULT_MIRROR_MAX_IMAGES 8 /* limited by kernel DM_KCOPYD_MAX_REGIONS */ +#define DEFAULT_RAID_FAULT_POLICY "warn" #define DEFAULT_DMEVENTD_RAID_LIB "libdevmapper-event-lvm2raid.so" #define DEFAULT_DMEVENTD_MIRROR_LIB "libdevmapper-event-lvm2mirror.so" #define DEFAULT_DMEVENTD_SNAPSHOT_LIB "libdevmapper-event-lvm2snapshot.so" diff --git a/tools/lvconvert.c b/tools/lvconvert.c index 1b9f6f9b..2b84de4c 100644 --- a/tools/lvconvert.c +++ b/tools/lvconvert.c @@ -1424,9 +1424,44 @@ static int is_valid_raid_conversion(const struct segment_type *from_segtype, return 1; } +static void _lvconvert_raid_repair_ask(struct cmd_context *cmd, int *replace_dev) +{ + const char *dev_policy = NULL; + + int force = arg_count(cmd, force_ARG); + int yes = arg_count(cmd, yes_ARG); + + *replace_dev = 0; + + if (arg_count(cmd, use_policies_ARG)) { + dev_policy = find_config_tree_str(cmd, "activation/raid_fault_policy", DEFAULT_RAID_FAULT_POLICY); + + if (!strcmp(dev_policy, "allocate") || + !strcmp(dev_policy, "replace")) + *replace_dev = 1; + /* else if (!strcmp(dev_policy, "anything_else")) -- ignore */ + return; + } + + if (yes) { + *replace_dev = 1; + return; + } + + if (force != PROMPT) + return; + + if (yes_no_prompt("Attempt to replace failed RAID images " + "(requires full device resync)? [y/n]: ") == 'y') { + *replace_dev = 1; + } +} + static int lvconvert_raid(struct logical_volume *lv, struct lvconvert_params *lp) { + int replace = 0; int image_count; + struct dm_list *failed_pvs; struct cmd_context *cmd = lv->vg->cmd; struct lv_segment *seg = first_seg(lv); @@ -1485,6 +1520,25 @@ static int lvconvert_raid(struct logical_volume *lv, struct lvconvert_params *lp if (arg_count(cmd, replace_ARG)) return lv_raid_replace(lv, lp->replace_pvh, lp->pvh); + if (arg_count(cmd, repair_ARG)) { + _lvconvert_raid_repair_ask(cmd, &replace); + + if (replace) { + if (!(failed_pvs = _failed_pv_list(lv->vg))) { + stack; + return ECMD_FAILED; + } + return lv_raid_replace(lv, failed_pvs, lp->pvh); + } + + /* "warn" if policy not set to replace */ + if (arg_count(cmd, use_policies_ARG)) + log_error("Issue 'lvconvert --repair %s/%s' to " + "replace failed device", + lv->vg->name, lv->name); + return 1; + } + log_error("Conversion operation not yet supported."); return 0; } |