--- libmultipath/checkers.c | 3 libmultipath/checkers.h | 9 + libmultipath/config.c | 4 libmultipath/config.h | 6 + libmultipath/configure.c | 2 libmultipath/defaults.h | 1 libmultipath/dict.c | 204 ++++++++++++++++++++++++++++++++++++++++++++- libmultipath/print.c | 2 libmultipath/propsel.c | 52 +++++++++++ libmultipath/propsel.h | 2 libmultipath/structs.h | 9 + multipath.conf.annotated | 40 ++++++++ multipath.conf.defaults | 2 multipath/multipath.conf.5 | 27 +++++ multipathd/main.c | 34 ++++++- 15 files changed, 388 insertions(+), 9 deletions(-) Index: multipath-tools-130222/libmultipath/config.h =================================================================== --- multipath-tools-130222.orig/libmultipath/config.h +++ multipath-tools-130222/libmultipath/config.h @@ -62,6 +62,8 @@ struct hwentry { int retain_hwhandler; int detect_prio; int deferred_remove; + int delay_watch_checks; + int delay_wait_checks; char * bl_product; }; @@ -86,6 +88,8 @@ struct mpentry { int attribute_flags; int user_friendly_names; int deferred_remove; + int delay_watch_checks; + int delay_wait_checks; uid_t uid; gid_t gid; mode_t mode; @@ -133,6 +137,8 @@ struct config { int deferred_remove; int ignore_new_boot_devs; int processed_main_config; + int delay_watch_checks; + int delay_wait_checks; unsigned int version[3]; char * dev; Index: multipath-tools-130222/libmultipath/structs.h =================================================================== --- multipath-tools-130222.orig/libmultipath/structs.h +++ multipath-tools-130222/libmultipath/structs.h @@ -134,6 +134,11 @@ enum scsi_protocol { SCSI_PROTOCOL_UNSPEC = 0xf, /* No specific protocol */ }; +enum delay_checks_states { + DELAY_CHECKS_OFF = -1, + DELAY_CHECKS_UNDEF = 0, +}; + struct sg_id { int host_no; int channel; @@ -180,6 +185,8 @@ struct path { int priority; int pgindex; int detect_prio; + int watch_checks; + int wait_checks; char * uid_attribute; struct prio prio; char * prio_args; @@ -215,6 +222,8 @@ struct multipath { int fast_io_fail; int retain_hwhandler; int deferred_remove; + int delay_watch_checks; + int delay_wait_checks; unsigned int dev_loss; uid_t uid; gid_t gid; Index: multipath-tools-130222/libmultipath/checkers.h =================================================================== --- multipath-tools-130222.orig/libmultipath/checkers.h +++ multipath-tools-130222/libmultipath/checkers.h @@ -46,6 +46,14 @@ * PATH_PENDING: * - Use: All async checkers * - Description: Indicates a check IO is in flight. + * + * PATH_DELAYED: + * - Use: None of the checkers (returned if the path is being delayed before + * reintegration. + * - Description: If a path fails after being up for less than + * delay_watch_checks checks, when it comes back up again, it will not + * be marked as up until it has been up for delay_wait_checks checks. + * During this time, it is marked as "delayed" */ enum path_check_state { PATH_WILD, @@ -55,6 +63,7 @@ enum path_check_state { PATH_SHAKY, PATH_GHOST, PATH_PENDING, + PATH_DELAYED, PATH_MAX_STATE }; Index: multipath-tools-130222/libmultipath/configure.c =================================================================== --- multipath-tools-130222.orig/libmultipath/configure.c +++ multipath-tools-130222/libmultipath/configure.c @@ -291,6 +291,8 @@ setup_map (struct multipath * mpp, char select_reservation_key(mpp); select_retain_hwhandler(mpp); select_deferred_remove(mpp); + select_delay_watch_checks(mpp); + select_delay_wait_checks(mpp); sysfs_set_scsi_tmo(mpp); /* Index: multipath-tools-130222/libmultipath/defaults.h =================================================================== --- multipath-tools-130222.orig/libmultipath/defaults.h +++ multipath-tools-130222/libmultipath/defaults.h @@ -20,6 +20,7 @@ #define DEFAULT_RETAIN_HWHANDLER RETAIN_HWHANDLER_OFF #define DEFAULT_DETECT_PRIO DETECT_PRIO_OFF #define DEFAULT_DEFERRED_REMOVE DEFERRED_REMOVE_OFF +#define DEFAULT_DELAY_CHECKS DELAY_CHECKS_OFF #define DEFAULT_CHECKINT 5 #define MAX_CHECKINT(a) (a << 2) Index: multipath-tools-130222/libmultipath/dict.c =================================================================== --- multipath-tools-130222.orig/libmultipath/dict.c +++ multipath-tools-130222/libmultipath/dict.c @@ -801,6 +801,44 @@ def_ignore_new_boot_devs_handler(vector return 0; } +static int +def_delay_watch_checks_handler(vector strvec) +{ + char * buff; + + buff = set_value(strvec); + if (!buff) + return 1; + + if ((strlen(buff) == 2 && !strcmp(buff, "no")) || + (strlen(buff) == 1 && !strcmp(buff, "0"))) + conf->delay_watch_checks = DELAY_CHECKS_OFF; + else if ((conf->delay_watch_checks = atoi(buff)) < 1) + conf->delay_watch_checks = DELAY_CHECKS_OFF; + + FREE(buff); + return 0; +} + +static int +def_delay_wait_checks_handler(vector strvec) +{ + char * buff; + + buff = set_value(strvec); + if (!buff) + return 1; + + if ((strlen(buff) == 2 && !strcmp(buff, "no")) || + (strlen(buff) == 1 && !strcmp(buff, "0"))) + conf->delay_wait_checks = DELAY_CHECKS_OFF; + else if ((conf->delay_wait_checks = atoi(buff)) < 1) + conf->delay_wait_checks = DELAY_CHECKS_OFF; + + FREE(buff); + return 0; +} + /* * blacklist block handlers */ @@ -1517,6 +1555,52 @@ hw_deferred_remove_handler(vector strvec return 0; } +static int +hw_delay_watch_checks_handler(vector strvec) +{ + struct hwentry *hwe = VECTOR_LAST_SLOT(conf->hwtable); + char * buff; + + if (!hwe) + return 1; + + buff = set_value(strvec); + if (!buff) + return 1; + + if ((strlen(buff) == 2 && !strcmp(buff, "no")) || + (strlen(buff) == 1 && !strcmp(buff, "0"))) + hwe->delay_watch_checks = DELAY_CHECKS_OFF; + else if ((hwe->delay_watch_checks = atoi(buff)) < 1) + hwe->delay_watch_checks = DELAY_CHECKS_OFF; + + FREE(buff); + return 0; +} + +static int +hw_delay_wait_checks_handler(vector strvec) +{ + struct hwentry *hwe = VECTOR_LAST_SLOT(conf->hwtable); + char * buff; + + if (!hwe) + return 1; + + buff = set_value(strvec); + if (!buff) + return 1; + + if ((strlen(buff) == 2 && !strcmp(buff, "no")) || + (strlen(buff) == 1 && !strcmp(buff, "0"))) + hwe->delay_wait_checks = DELAY_CHECKS_OFF; + else if ((hwe->delay_wait_checks = atoi(buff)) < 1) + hwe->delay_wait_checks = DELAY_CHECKS_OFF; + + FREE(buff); + return 0; +} + /* * multipaths block handlers */ @@ -1996,6 +2080,52 @@ mp_deferred_remove_handler(vector strvec return 0; } +static int +mp_delay_watch_checks_handler(vector strvec) +{ + struct mpentry *mpe = VECTOR_LAST_SLOT(conf->mptable); + char * buff; + + if (!mpe) + return 1; + + buff = set_value(strvec); + if (!buff) + return 1; + + if ((strlen(buff) == 2 && !strcmp(buff, "no")) || + (strlen(buff) == 1 && !strcmp(buff, "0"))) + mpe->delay_watch_checks = DELAY_CHECKS_OFF; + else if ((mpe->delay_watch_checks = atoi(buff)) < 1) + mpe->delay_watch_checks = DELAY_CHECKS_OFF; + + FREE(buff); + return 0; +} + +static int +mp_delay_wait_checks_handler(vector strvec) +{ + struct mpentry *mpe = VECTOR_LAST_SLOT(conf->mptable); + char * buff; + + if (!mpe) + return 1; + + buff = set_value(strvec); + if (!buff) + return 1; + + if ((strlen(buff) == 2 && !strcmp(buff, "no")) || + (strlen(buff) == 1 && !strcmp(buff, "0"))) + mpe->delay_wait_checks = DELAY_CHECKS_OFF; + else if ((mpe->delay_wait_checks = atoi(buff)) < 1) + mpe->delay_wait_checks = DELAY_CHECKS_OFF; + + FREE(buff); + return 0; +} + /* * config file keywords printing */ @@ -2258,6 +2388,30 @@ snprint_mp_deferred_remove (char * buff, } static int +snprint_mp_delay_watch_checks(char * buff, int len, void * data) +{ + struct mpentry * mpe = (struct mpentry *)data; + + if (mpe->delay_watch_checks == DELAY_CHECKS_UNDEF) + return 0; + if (mpe->delay_watch_checks == DELAY_CHECKS_OFF) + return snprintf(buff, len, "no"); + return snprintf(buff, len, "%d", mpe->delay_watch_checks); +} + +static int +snprint_mp_delay_wait_checks(char * buff, int len, void * data) +{ + struct mpentry * mpe = (struct mpentry *)data; + + if (mpe->delay_wait_checks == DELAY_CHECKS_UNDEF) + return 0; + if (mpe->delay_wait_checks == DELAY_CHECKS_OFF) + return snprintf(buff, len, "no"); + return snprintf(buff, len, "%d", mpe->delay_wait_checks); +} + +static int snprint_hw_fast_io_fail(char * buff, int len, void * data) { struct hwentry * hwe = (struct hwentry *)data; @@ -2586,6 +2740,30 @@ snprint_hw_deferred_remove(char * buff, } static int +snprint_hw_delay_watch_checks(char * buff, int len, void * data) +{ + struct hwentry * hwe = (struct hwentry *)data; + + if (hwe->delay_watch_checks == DELAY_CHECKS_UNDEF) + return 0; + if (hwe->delay_watch_checks == DELAY_CHECKS_OFF) + return snprintf(buff, len, "no"); + return snprintf(buff, len, "%d", hwe->delay_watch_checks); +} + +static int +snprint_hw_delay_wait_checks(char * buff, int len, void * data) +{ + struct hwentry * hwe = (struct hwentry *)data; + + if (hwe->delay_wait_checks == DELAY_CHECKS_UNDEF) + return 0; + if (hwe->delay_wait_checks == DELAY_CHECKS_OFF) + return snprintf(buff, len, "no"); + return snprintf(buff, len, "%d", hwe->delay_wait_checks); +} + +static int snprint_detect_prio(char * buff, int len, void * data) { struct hwentry * hwe = (struct hwentry *)data; @@ -2883,7 +3061,6 @@ snprint_def_find_multipaths (char * buff return snprintf(buff, len, "yes"); } - static int snprint_def_user_friendly_names (char * buff, int len, void * data) { @@ -2989,7 +3166,6 @@ snprint_def_ignore_new_boot_devs(char * return snprintf(buff, len, "no"); } - static int snprint_def_config_dir (char * buff, int len, void * data) { @@ -3000,6 +3176,24 @@ snprint_def_config_dir (char * buff, int } static int +snprint_def_delay_watch_checks(char * buff, int len, void * data) +{ + if (conf->delay_watch_checks == DELAY_CHECKS_UNDEF || + conf->delay_watch_checks == DELAY_CHECKS_OFF) + return snprintf(buff, len, "no"); + return snprintf(buff, len, "%d", conf->delay_watch_checks); +} + +static int +snprint_def_delay_wait_checks(char * buff, int len, void * data) +{ + if (conf->delay_wait_checks == DELAY_CHECKS_UNDEF || + conf->delay_wait_checks == DELAY_CHECKS_OFF) + return snprintf(buff, len, "no"); + return snprintf(buff, len, "%d", conf->delay_wait_checks); +} + +static int snprint_ble_simple (char * buff, int len, void * data) { struct blentry * ble = (struct blentry *)data; @@ -3071,6 +3265,8 @@ init_keywords(void) install_keyword("deferred_remove", &def_deferred_remove_handler, &snprint_def_deferred_remove); install_keyword("ignore_new_boot_devs", &def_ignore_new_boot_devs_handler, &snprint_def_ignore_new_boot_devs); install_keyword("config_dir", &def_config_dir_handler, &snprint_def_config_dir); + install_keyword("delay_watch_checks", &def_delay_watch_checks_handler, &snprint_def_delay_watch_checks); + install_keyword("delay_wait_checks", &def_delay_wait_checks_handler, &snprint_def_delay_wait_checks); __deprecated install_keyword("default_selector", &def_selector_handler, NULL); __deprecated install_keyword("default_path_grouping_policy", &def_pgpolicy_handler, NULL); __deprecated install_keyword("default_uid_attribute", &def_uid_attribute_handler, NULL); @@ -3136,6 +3332,8 @@ init_keywords(void) install_keyword("retain_attached_hw_handler", &hw_retain_hwhandler_handler, &snprint_hw_retain_hwhandler_handler); install_keyword("detect_prio", &hw_detect_prio_handler, &snprint_detect_prio); install_keyword("deferred_remove", &hw_deferred_remove_handler, &snprint_hw_deferred_remove); + install_keyword("delay_watch_checks", &hw_delay_watch_checks_handler, &snprint_hw_delay_watch_checks); + install_keyword("delay_wait_checks", &hw_delay_wait_checks_handler, &snprint_hw_delay_wait_checks); install_sublevel_end(); install_keyword_root("multipaths", &multipaths_handler); @@ -3161,5 +3359,7 @@ init_keywords(void) install_keyword("reservation_key", &mp_reservation_key_handler, &snprint_mp_reservation_key); install_keyword("user_friendly_names", &mp_names_handler, &snprint_mp_user_friendly_names); install_keyword("deferred_remove", &mp_deferred_remove_handler, &snprint_mp_deferred_remove); + install_keyword("delay_watch_checks", &mp_delay_watch_checks_handler, &snprint_mp_delay_watch_checks); + install_keyword("delay_wait_checks", &mp_delay_wait_checks_handler, &snprint_mp_delay_wait_checks); install_sublevel_end(); } Index: multipath-tools-130222/libmultipath/print.c =================================================================== --- multipath-tools-130222.orig/libmultipath/print.c +++ multipath-tools-130222/libmultipath/print.c @@ -336,6 +336,8 @@ snprint_chk_state (char * buff, size_t l return snprintf(buff, len, "shaky"); case PATH_GHOST: return snprintf(buff, len, "ghost"); + case PATH_DELAYED: + return snprintf(buff, len, "delayed"); default: return snprintf(buff, len, "undef"); } Index: multipath-tools-130222/libmultipath/propsel.c =================================================================== --- multipath-tools-130222.orig/libmultipath/propsel.c +++ multipath-tools-130222/libmultipath/propsel.c @@ -788,3 +788,55 @@ select_detect_prio (struct path * pp) condlog(3, "%s: detect_prio = %d (compiled in default)", pp->dev, pp->detect_prio); return 0; } + +extern int +select_delay_watch_checks (struct multipath * mp) +{ + if (mp->mpe && mp->mpe->delay_watch_checks != DELAY_CHECKS_UNDEF) { + mp->delay_watch_checks = mp->mpe->delay_watch_checks; + condlog(3, "delay_watch_checks = %i (multipath setting)", + mp->delay_watch_checks); + return 0; + } + if (mp->hwe && mp->hwe->delay_watch_checks != DELAY_CHECKS_UNDEF) { + mp->delay_watch_checks = mp->hwe->delay_watch_checks; + condlog(3, "delay_watch_checks = %i (controler setting)", + mp->delay_watch_checks); + return 0; + } + if (conf->delay_watch_checks != DELAY_CHECKS_UNDEF) { + mp->delay_watch_checks = conf->delay_watch_checks; + condlog(3, "delay_watch_checks = %i (config file default)", + mp->delay_watch_checks); + return 0; + } + mp->delay_watch_checks = DEFAULT_DELAY_CHECKS; + condlog(3, "delay_watch_checks = DISABLED (internal default)"); + return 0; +} + +extern int +select_delay_wait_checks (struct multipath * mp) +{ + if (mp->mpe && mp->mpe->delay_wait_checks != DELAY_CHECKS_UNDEF) { + mp->delay_wait_checks = mp->mpe->delay_wait_checks; + condlog(3, "delay_wait_checks = %i (multipath setting)", + mp->delay_wait_checks); + return 0; + } + if (mp->hwe && mp->hwe->delay_wait_checks != DELAY_CHECKS_UNDEF) { + mp->delay_wait_checks = mp->hwe->delay_wait_checks; + condlog(3, "delay_wait_checks = %i (controler setting)", + mp->delay_wait_checks); + return 0; + } + if (conf->delay_wait_checks != DELAY_CHECKS_UNDEF) { + mp->delay_wait_checks = conf->delay_wait_checks; + condlog(3, "delay_wait_checks = %i (config file default)", + mp->delay_wait_checks); + return 0; + } + mp->delay_wait_checks = DEFAULT_DELAY_CHECKS; + condlog(3, "delay_wait_checks = DISABLED (internal default)"); + return 0; +} Index: multipath-tools-130222/libmultipath/propsel.h =================================================================== --- multipath-tools-130222.orig/libmultipath/propsel.h +++ multipath-tools-130222/libmultipath/propsel.h @@ -21,3 +21,5 @@ int select_reservation_key(struct multip int select_retain_hwhandler (struct multipath * mp); int select_detect_prio(struct path * pp); int select_deferred_remove(struct multipath *mp); +int select_delay_watch_checks (struct multipath * mp); +int select_delay_wait_checks (struct multipath * mp); Index: multipath-tools-130222/multipathd/main.c =================================================================== --- multipath-tools-130222.orig/multipathd/main.c +++ multipath-tools-130222/multipathd/main.c @@ -188,7 +188,8 @@ sync_map_state(struct multipath *mpp) vector_foreach_slot (mpp->pg, pgp, i){ vector_foreach_slot (pgp->paths, pp, j){ if (pp->state == PATH_UNCHECKED || - pp->state == PATH_WILD) + pp->state == PATH_WILD || + pp->state == PATH_DELAYED) continue; if ((pp->dmstate == PSTATE_FAILED || pp->dmstate == PSTATE_UNDEF) && @@ -1165,6 +1166,16 @@ check_path (struct vectors * vecs, struc if (!pp->mpp) return; + if ((newstate == PATH_UP || newstate == PATH_GHOST) && + pp->wait_checks > 0) { + if (pp->mpp && pp->mpp->nr_active > 0) { + pp->state = PATH_DELAYED; + pp->wait_checks--; + return; + } else + pp->wait_checks = 0; + } + pp->chkrstate = newstate; if (newstate != pp->state) { int oldstate = pp->state; @@ -1182,9 +1193,14 @@ check_path (struct vectors * vecs, struc * proactively fail path in the DM */ if (oldstate == PATH_UP || - oldstate == PATH_GHOST) + oldstate == PATH_GHOST) { fail_path(pp, 1); - else + if (pp->mpp->delay_wait_checks > 0 && + pp->watch_checks > 0) { + pp->wait_checks = pp->mpp->delay_wait_checks; + pp->watch_checks = 0; + } + }else fail_path(pp, 0); /* @@ -1211,11 +1227,15 @@ check_path (struct vectors * vecs, struc * reinstate this path */ if (oldstate != PATH_UP && - oldstate != PATH_GHOST) + oldstate != PATH_GHOST) { + if (pp->mpp->delay_watch_checks > 0) + pp->watch_checks = pp->mpp->delay_watch_checks; reinstate_path(pp, 1); - else + } else { + if (pp->watch_checks > 0) + pp->watch_checks--; reinstate_path(pp, 0); - + } new_path_up = 1; if (oldchkrstate != PATH_UP && oldchkrstate != PATH_GHOST) @@ -1245,6 +1265,8 @@ check_path (struct vectors * vecs, struc else pp->checkint = conf->max_checkint; } + if (pp->watch_checks > 0) + pp->watch_checks--; pp->tick = pp->checkint; condlog(4, "%s: delay next check %is", pp->dev_t, pp->tick); Index: multipath-tools-130222/multipath.conf.annotated =================================================================== --- multipath-tools-130222.orig/multipath.conf.annotated +++ multipath-tools-130222/multipath.conf.annotated @@ -242,6 +242,30 @@ # # files, just as if it was in /etc/multipath.conf # # values : "" or a fully qualified pathname # # default : "/etc/multipath/conf.d" +# +# # +# # name : delay_watch_checks +# # scope : multipathd +# # desc : If set to a value greater than 0, multipathd will watch +# # paths that have recently become valid for this many +# # checks. If they fail again while they are being watched, +# # when they next become valid, they will not be used until +# # they have stayed up for delay_wait_checks checks. +# # values : no| > 0 +# # default : no +# delay_watch_checks 12 +# +# # +# # name : delay_wait_checks +# # scope : multipathd +# # desc : If set to a value greater than 0, when a device that has +# # recently come back online fails again within +# # delay_watch_checks checks, the next time it comes back +# # online, it will marked and delayed, and not used until +# # it has passed delay_wait_checks checks. +# # values : no| > 0 +# # default : no +# delay_wait_checks 12 #} # ## @@ -383,6 +407,13 @@ # # # flush_on_last_del yes # +# # +# # name : delay_watch_checks +# # See defualts section for information. +# +# # +# # name : delay_wait_checks +# # See defualts section for information. # } # multipath { # wwid 1DEC_____321816758474 @@ -566,6 +597,15 @@ # # before removing it from the system. # # values : n > 0 # dev_loss_tmo 600 +# +# # +# # name : delay_watch_checks +# # See defaults section for information. +# +# # +# # name : delay_wait_checks +# # See defaults section for information. +# # } # device { # vendor "COMPAQ " Index: multipath-tools-130222/multipath.conf.defaults =================================================================== --- multipath-tools-130222.orig/multipath.conf.defaults +++ multipath-tools-130222/multipath.conf.defaults @@ -27,6 +27,8 @@ # retain_attached_hw_handler no # detect_prio no # config_dir "/etc/multipath/conf.d" +# delay_watch_checks no +# delay_wait_checks no #} #blacklist { # devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*" Index: multipath-tools-130222/multipath/multipath.conf.5 =================================================================== --- multipath-tools-130222.orig/multipath/multipath.conf.5 +++ multipath-tools-130222/multipath/multipath.conf.5 @@ -459,6 +459,25 @@ alphabetically for file ending in ".conf information from them, just as if it was in /etc/multipath.conf. config_dir must either be "" or a fully qualified directory name. Default is .I "/etc/multipath/conf.d" +.TP +.B delay_watch_checks +If set to a value greater than 0, multipathd will watch paths that have +recently become valid for this many checks. If they fail again while they are +being watched, when they next become valid, they will not be used until they +have stayed up for +.I delay_wait_checks +checks. Default is +.I no +.TP +.B delay_wait_checks +If set to a value greater than 0, when a device that has recently come back +online fails again within +.I delay_watch_checks +checks, the next time it comes back online, it will marked and delayed, and not +used until it has passed +.I delay_wait_checks +checks. Default is +.I no . .SH "blacklist section" The @@ -562,6 +581,10 @@ section: .B reservation_key .TP .B deferred_remove +.TP +.B delay_watch_checks +.TP +.B delay_wait_checks .RE .PD .LP @@ -654,6 +677,10 @@ section: .B detect_prio .TP .B deferred_remove +.TP +.B delay_watch_checks +.TP +.B delay_wait_checks .RE .PD .LP Index: multipath-tools-130222/libmultipath/checkers.c =================================================================== --- multipath-tools-130222.orig/libmultipath/checkers.c +++ multipath-tools-130222/libmultipath/checkers.c @@ -16,7 +16,8 @@ char *checker_state_names[] = { "up", "shaky", "ghost", - "pending" + "pending", + "delayed" }; static LIST_HEAD(checkers); Index: multipath-tools-130222/libmultipath/config.c =================================================================== --- multipath-tools-130222.orig/libmultipath/config.c +++ multipath-tools-130222/libmultipath/config.c @@ -341,6 +341,8 @@ merge_hwe (struct hwentry * dst, struct merge_num(retain_hwhandler); merge_num(detect_prio); merge_num(deferred_remove); + merge_num(delay_watch_checks); + merge_num(delay_wait_checks); /* * Make sure features is consistent with @@ -399,6 +401,8 @@ overwrite_hwe (struct hwentry * dst, str overwrite_num(retain_hwhandler); overwrite_num(detect_prio); overwrite_num(deferred_remove); + overwrite_num(delay_watch_checks); + overwrite_num(delay_wait_checks); /* * Make sure features is consistent with