Automatic brick ordering and finite state machine support.

This patch adds preliminary FSM support. This will be used and abused
more extensively in later patches. Automatic brick ordering is an
advanced feature and is meant for experienced puppet users. Changing the
available bricks before the cluster is built is not currently supported.
For that type of magic, please wait for gluster::elastic.

This feature expects that you name your bricks and hosts intelligently.
Future patches will recommend a specific nomenclature, but for now as
long as the brick paths and hostnames follow a padded, incrementing
integer pattern, with a common prefix, you shouldn't see any problems.

1 files changed, 174 insertions, 19 deletions

diff --git a/manifests/volume.pp b/manifests/volume.pp
index 8c79dd8..5e8b7e1 100644
--- a/manifests/volume.pp
+++ b/manifests/volume.pp
@@ -16,12 +16,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 define gluster::volume(
-	$bricks = [],
+	$bricks = true,		# specify a list of bricks, or true for auto...
+	$group = 'default',	# use this bricks group name if we auto collect
 	$transport = 'tcp',
 	$replica = 1,
 	$stripe = 1,
 	$vip = '',		# vip of the cluster (optional but recommended)
 	$ping = true,		# do we want to include fping checks ?
+	$settle = true,		# do we want to run settle checks ?
 	$start = undef		# start volume ? true, false (stop it) or undef
 ) {
 	include gluster::xml
@@ -34,6 +36,95 @@ define gluster::volume(
 	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
 	$vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '')
 
+	$settle_count = 3		# three is a reasonable default!
+	$maxlength = 3
+	if $maxlength < $settle_count {
+		fail('The $maxlength needs to be greater than or equal to the $settle_count.')
+	}
+	$are_bricks_collected = (type($bricks) == 'boolean' and ($bricks == true))
+	# NOTE: settle checks are still useful even if we are using ping checks
+	# the reason why they are still useful, is that they can detect changes
+	# in the bricks, which might propagate slowly because of exported types
+	# the fping checks can only verify that the individual hosts are alive!
+	$settle_count_check = $are_bricks_collected ? {
+		false => false,
+		default => $settle ? {
+			false => false,
+			default => true,
+		}
+	}
+	# TODO: implement settle_time_check
+	$settle_time_check = $settle ? {
+		false => false,
+		default => true,
+	}
+
+	# clean up old fsm data when not in use, because parent $vardir purges!
+	if $are_bricks_collected {
+		include gluster::volume::fsm
+		file { "${vardir}/volume/fsm/${name}/":
+			ensure => directory,	# make sure this is a directory
+			recurse => true,	# recurse into directory
+			purge => false,		# don't purge unmanaged files
+			force => false,		# don't purge subdirs and links
+			require => File["${vardir}/volume/fsm/"],
+		}
+	}
+
+	$collected_bricks = split(getvar("gluster_brick_group_${group}"), ',')
+	$valid_bricks = type($bricks) ? {
+		'boolean' => $bricks ? {
+			true => $collected_bricks,		# an array...
+			default => [],				# invalid type
+		},
+		'list' => $bricks,
+		default => [],					# invalid type
+	}
+
+	# helpful debugging!
+	notice(inline_template('valid_bricks: <%= valid_bricks.inspect %>'))
+
+	# NOTE: we're using the valid_bricks value here, and not the collected
+	# value. while we only need the $collected value for settle detection,
+	# the actual brick value is needed for future add/remove brick code...
+	$valid_input = join($valid_bricks, ',')	# TODO: this should be pickled
+	if $are_bricks_collected and "${valid_input}" == '' {
+		notice('The gluster::brick collection is not ready yet.')
+	}
+	$last = getvar("gluster_volume_fsm_state_${name}")	# fact !
+	$valid_last = "${last}" ? {
+		# initialize the $last var to match the $input if it's empty...
+		'' => "${valid_input}",
+		default => "${last}",
+	}
+	if $are_bricks_collected and ("${valid_input}" != '') and ("${valid_last}" == '') {
+		fail('Previous state is invalid.')	# fact was tampered with
+	}
+
+	# NOTE: the stack lists are left base64 encoded since we only care if they change! :P
+	# NOTE: each element in stack is base64 encoded because they contain: ,
+	$stack_fact = getvar("gluster_volume_fsm_stack_${name}")	# fact !
+	$stack_full = split("${stack_fact}", ',')
+	$stack_trim = "${maxlength}" ? {
+		'-1' => $stack_full,	# unlimited
+		#default => split(inline_template('<%= stack_full[0,maxlength.to_i.abs].join(",") %>'), ','),
+		default => split(inline_template('<%= stack_full[[stack_full.size-maxlength.to_i.abs,0].max,maxlength.to_i.abs].join(",") %>'), ','),
+	}
+
+	$watch_fact = getvar("gluster_volume_fsm_watch_${name}")	# fact !
+	$watch_full = split("${watch_fact}", ',')
+	$watch_trim = "${maxlength}" ? {
+		'-1' => $watch_full,	# unlimited
+		#default => split(inline_template('<%= watch_full[0,maxlength.to_i.abs].join(",") %>'), ','),
+		default => split(inline_template('<%= watch_full[[watch_full.size-maxlength.to_i.abs,0].max,maxlength.to_i.abs].join(",") %>'), ','),
+	}
+
+	# if the last $settle_count elements are the same, the template
+	# should reduce down to the value '1'. check this and the size.
+	$one = inline_template('<%= watch_trim[[watch_trim.size-settle_count.to_i,0].max,settle_count.to_i].uniq.size %>')
+	$watch_trim_size = size($watch_trim)
+	$settled = ((! $settle_count_check) or ((size($watch_trim) >= $settle_count) and "${one}" == '1'))
+
 	# TODO: if using rdma, maybe we should pull in the rdma package... ?
 	$valid_transport = $transport ? {
 		'rdma' => 'rdma',
@@ -59,24 +150,24 @@ define gluster::volume(
 	# returns interface name that has vip, or '' if none are found.
 	$vipif = inline_template("<%= interfaces.split(',').find_all {|x| '${valid_vip}' == scope.lookupvar('ipaddress_'+x) }[0,1].join('') %>")
 
-	#Gluster::Brick[$bricks] -> Gluster::Volume[$name]	# volume requires bricks
+	#Gluster::Brick[$valid_bricks] -> Gluster::Volume[$name]	# volume requires bricks
 
 	# get the bricks that match our fqdn, and append /$name to their path.
 	# return only these paths, which can be used to build the volume dirs.
 	# NOTE: gluster v3.4 won't create a volume if this dir already exists.
 	# TODO: is this needed when bricks are devices and not on filesystem ?
-	#$volume_dirs = split(inline_template("<%= bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}' }.join(' ') %>"), ' ')
+	#$volume_dirs = split(inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}' }.join(' ') %>"), ' ')
 	#file { $volume_dirs:
 	#	ensure => directory,		# make sure this is a directory
 	#	recurse => false,		# don't recurse into directory
 	#	purge => false,			# don't purge unmanaged files
 	#	force => false,			# don't purge subdirs and links
 	#	before => Exec["gluster-volume-create-${name}"],
-	#	require => Gluster::Brick[$bricks],
+	#	require => Gluster::Brick[$valid_bricks],
 	#}
 
 	# add /${name} to the end of each: brick:/path entry
-	$brick_spec = inline_template("<%= bricks.collect {|x| ''+x.chomp('/')+'/${name}' }.join(' ') %>")
+	$brick_spec = inline_template("<%= valid_bricks.collect {|x| ''+x.chomp('/')+'/${name}' }.join(' ') %>")
 
 	# if volume creation fails for a stupid reason, in many cases, glusterd
 	# already did some of the work and left us with volume name directories
@@ -85,10 +176,10 @@ define gluster::volume(
 	# we error we should rmdir any empty volume dirs to keep it pristine...
 	# TODO: this should be a gluster bug... we must hope it doesn't happen!
 	# maybe related to: https://bugzilla.redhat.com/show_bug.cgi?id=835494
-	$rmdir_volume_dirs = sprintf("/bin/rmdir '%s'", inline_template("<%= bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}/' }.join('\' \'') %>"))
+	$rmdir_volume_dirs = sprintf("/bin/rmdir '%s'", inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}/' }.join('\' \'') %>"))
 
 	# get the list of bricks fqdn's that don't have our fqdn
-	$others = inline_template("<%= bricks.find_all{|x| x.split(':')[0] != '${fqdn}' }.collect {|y| y.split(':')[0] }.join(' ') %>")
+	$others = inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] != '${fqdn}' }.collect {|y| y.split(':')[0] }.join(' ') %>")
 
 	$fping = sprintf("/usr/sbin/fping -q %s", $others)
 	$status = sprintf("/usr/sbin/gluster peer status --xml | ${vardir}/xml.py connected %s", $others)
@@ -106,14 +197,14 @@ define gluster::volume(
 			Service['glusterd'],
 			File["${vardir}/volume/create-${name}.sh"],
 			File["${vardir}/xml.py"],	# status check
-			Gluster::Brick[$bricks],
+			Gluster::Brick[$valid_bricks],
 		],
 		default => [
 			Service['glusterd'],
 			File["${vardir}/volume/create-${name}.sh"],
 			Package['fping'],
 			File["${vardir}/xml.py"],	# status check
-			Gluster::Brick[$bricks],
+			Gluster::Brick[$valid_bricks],
 		],
 	}
 
@@ -145,14 +236,17 @@ define gluster::volume(
 		# NOTE: in this command, we save the std{out,err} and pass them
 		# on too for puppet to consume. we save in /tmp for fast access
 		# EXAMPLE: gluster volume create test replica 2 transport tcp annex1.example.com:/storage1a/test annex2.example.com:/storage2a/test annex3.example.com:/storage3b/test annex4.example.com:/storage4b/test annex1.example.com:/storage1c/test annex2.example.com:/storage2c/test annex3.example.com:/storage3d/test annex4.example.com:/storage4d/test
-		exec { "gluster-volume-create-${name}":
-			command => "${vardir}/volume/create-${name}.sh",
-			logoutput => on_failure,
-			unless => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -",	# add volume if it doesn't exist
-			onlyif => $onlyif,
-			#before => TODO?,
-			require => $require,
-			alias => "gluster-volume-create-${name}",
+		# TODO: add a timer similar to my puppet-runonce timer code... to wait X minutes after we've settled... useful if we Exec['again'] to speed things up...
+		if $settled {
+			exec { "gluster-volume-create-${name}":
+				command => "${vardir}/volume/create-${name}.sh",
+				logoutput => on_failure,
+				unless => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -",	# add volume if it doesn't exist
+				onlyif => $onlyif,
+				#before => TODO?,
+				require => $require,
+				alias => "gluster-volume-create-${name}",
+			}
 		}
 	}
 
@@ -164,7 +258,10 @@ define gluster::volume(
 				logoutput => on_failure,
 				onlyif => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -",
 				unless => "/usr/sbin/gluster volume status ${name}",	# returns false if stopped
-				require => Exec["gluster-volume-create-${name}"],
+				require => $settled ? {	# require if type exists
+					false => undef,
+					default => Exec["gluster-volume-create-${name}"],
+				},
 				alias => "gluster-volume-start-${name}",
 			}
 		} elsif ( $start == false ) {
@@ -177,7 +274,10 @@ define gluster::volume(
 			exec { "/usr/bin/yes | /usr/sbin/gluster volume stop ${name}":
 				logoutput => on_failure,
 				onlyif => "/usr/sbin/gluster volume status ${name}",	# returns true if started
-				require => Exec["gluster-volume-create-${name}"],
+				require => $settled ? {	# require if type exists
+					false => undef,
+					default => Exec["gluster-volume-create-${name}"],
+				},
 				alias => "gluster-volume-stop-${name}",
 			}
 		} else {	# 'undef'-ined
@@ -218,6 +318,61 @@ define gluster::volume(
 			source => "${zone}",	# use our source zone
 			before => Service['glusterd'],
 		}
+
+	# fsm variables and boilerplate
+	$statefile = "${vardir}/volume/fsm/${name}/state"
+	$stackfile = "${vardir}/volume/fsm/${name}/stack"
+	$watchfile = "${vardir}/volume/fsm/${name}/watch"
+	$diff = "/usr/bin/test '${valid_input}' != '${valid_last}'"
+	$stack_truncate = "${maxlength}" ? {
+		'-1' => '',	# unlimited
+		#default => sprintf("&& /bin/sed -i '%d,$ d' ${stackfile}", inline_template('<%= maxlength.to_i.abs+1 %>')),
+		default => sprintf(" && (/bin/grep -v '^$' ${stackfile} | /usr/bin/tail -%d | /usr/bin/tee ${stackfile})", inline_template('<%= maxlength.to_i.abs %>')),
+	}
+	$watch_truncate = "${maxlength}" ? {
+		'-1' => '',	# unlimited
+		#default => sprintf("&& /bin/sed -i '%d,$ d' ${watchfile}", inline_template('<%= maxlength.to_i.abs+1 %>')),
+		default => sprintf(" && (/bin/grep -v '^$' ${watchfile} | /usr/bin/tail -%d | /usr/bin/tee ${watchfile})", inline_template('<%= maxlength.to_i.abs %>')),
+	}
+
+	if $are_bricks_collected and ("${valid_input}" != '') {	# ready or not?
+
+		# TODO: future versions should pickle (but with yaml)
+		exec { "/bin/echo '${valid_input}' > '${statefile}'":
+			logoutput => on_failure,
+			onlyif => "/usr/bin/test ! -e '${statefile}' || ${diff}",
+			require => File["${vardir}/volume/fsm/${name}/"],
+			alias => "gluster-volume-fsm-state-${name}",
+		}
+
+		# NOTE: keep a stack of past transitions, and load them in as a list...
+		exec { "/bin/echo '${valid_input}' >> '${stackfile}'${stack_truncate}":
+			logoutput => on_failure,
+			onlyif => "/usr/bin/test ! -e '${stackfile}' || ${diff}",
+			require => [
+				File["${vardir}/volume/fsm/${name}/"],
+				# easy way to ensure the transition types don't need to
+				# add a before to both exec's since this one follows it
+				Exec["gluster-volume-fsm-state-${name}"],
+			],
+			alias => "gluster-volume-fsm-stack-${name}",
+		}
+
+		# NOTE: watch *all* transitions, and load them in as a list...
+		exec { "/bin/echo '${valid_input}' >> '${watchfile}'${watch_truncate}":
+			logoutput => on_failure,
+			# we run this if the file doesn't exist, or there is a
+			# difference to record, or the sequence hasn't settled
+			# we also check that we have our minimum settle count!
+			onlyif => "/usr/bin/test ! -e '${watchfile}' || ${diff} || /usr/bin/test '1' != '${one}' || /usr/bin/test ${watch_trim_size} -lt ${settle_count}",
+			require => [
+				File["${vardir}/volume/fsm/${name}/"],
+				# easy way to ensure the transition types don't need to
+				# add a before to both exec's since this one follows it
+				Exec["gluster-volume-fsm-state-${name}"],
+			],
+			alias => "gluster-volume-fsm-watch-${name}",
+		}
 	}
 }