Automatic brick ordering and finite state machine support.

This patch adds preliminary FSM support. This will be used and abused
more extensively in later patches. Automatic brick ordering is an
advanced feature and is meant for experienced puppet users. Changing the
available bricks before the cluster is built is not currently supported.
For that type of magic, please wait for gluster::elastic.

This feature expects that you name your bricks and hosts intelligently.
Future patches will recommend a specific nomenclature, but for now as
long as the brick paths and hostnames follow a padded, incrementing
integer pattern, with a common prefix, you shouldn't see any problems.

9 files changed, 672 insertions, 20 deletions

diff --git a/examples/gluster-simple-example.pp b/examples/gluster-simple-example.pp
new file mode 100644
index 0000000..a050343
--- /dev/null
+++ b/examples/gluster-simple-example.pp
@@ -0,0 +1,16 @@
+#
+#	simple gluster setup. yeah, that's it.
+#	this should run on *every* gluster host
+#	NOTE: this should use a VIP.
+#
+
+node /^annex\d+$/ {	# annex{1,2,..N}
+
+	# NOTE: this class offers some configuration, see the source for info.
+	# NOTE: this is mostly intended for fast gluster testing. for more
+	# complex setups, you might want to look at the other examples.
+	class { '::gluster::simple':
+	}
+
+}
+
diff --git a/lib/facter/gluster_bricks.rb b/lib/facter/gluster_bricks.rb
new file mode 100644
index 0000000..b8852bc
--- /dev/null
+++ b/lib/facter/gluster_bricks.rb
@@ -0,0 +1,148 @@
+# GlusterFS module by James
+# Copyright (C) 2010-2013+ James Shubin
+# Written by James Shubin <james@shubin.ca>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'facter'
+
+# regexp to match a brick pattern, eg: annex1.example.com:/storage1a
+regexp = /^[a-zA-Z]{1}[a-zA-Z0-9\.\-]{0,}:\/[a-zA-Z0-9]{1}[a-zA-Z0-9\/\.\-]{0,}$/	# TODO: is this right ?
+
+# find the module_vardir
+dir = Facter.value('puppet_vardirtmp')		# nil if missing
+if dir.nil?					# let puppet decide if present!
+	dir = Facter.value('puppet_vardir')
+	if dir.nil?
+		var = nil
+	else
+		var = dir.gsub(/\/$/, '')+'/'+'tmp/'	# ensure trailing slash
+	end
+else
+	var = dir.gsub(/\/$/, '')+'/'
+end
+
+if var.nil?
+	# if we can't get a valid vardirtmp, then we can't continue
+	uuidfile = nil
+else
+	module_vardir = var+'gluster/'
+	valid_brickdir = module_vardir.gsub(/\/$/, '')+'/brick/'
+end
+
+found = {}
+result = {}
+
+if not(valid_brickdir.nil?) and File.directory?(valid_brickdir)
+	Dir.glob(valid_brickdir+'*.*').each do |f|
+		b = File.basename(f)
+		g = b.split('.')	# $name.group
+
+		group = g.pop()		# pop off suffix (the group name)
+		if not found.key?(group)
+			found[group] = []	# initialize
+		end
+
+		if g.length >= 1
+			x = g.join('.')	# in case value had dots in it.
+
+			brick = File.open(f, 'r').read.strip	# read into str
+			# eg: annex1.example.com:/storage1a
+			split = brick.split(':')	# do some $name parsing
+			host = split[0]			# host fqdn
+			# NOTE: technically $path should be everything BUT split[0]. This
+			# lets our $path include colons if for some reason they're needed.
+			#path = split[1]		# brick mount or storage path
+			path = brick.slice(host.length+1, brick.length-host.length-1)
+
+			if brick.length > 0 and regexp.match(brick)
+				found[group].push({'host' => host, 'path' => path})
+			# TODO: print warning on else...
+			end
+		end
+	end
+end
+
+# sort the bricks in a logical manner... i think this is the optimal algorithm,
+# but i'd be happy if someone thinks they can do better! this assumes that the
+# bricks and hosts are named in a logical manner. alphanumeric sorting is used
+# to determine the default ordering...
+# TODO: maybe this should be a puppet function instead of a fact... that way,
+# if necessary, the function could also include the replica count, and other
+# data as input... is it even needed ?
+
+found.keys.each do |group|
+
+	collect = {}
+	found[group].each do |x|
+		key = x['host']
+		val = x['path']
+
+		if not collect.has_key?(key)
+			collect[key] = []	# initialize
+		end
+
+		collect[key].push(val)	# save in array
+		# TODO: ensure this array is always sorted (we could also do this after
+		# or always insert elements in the correct sorted order too :P)
+		collect[key] = collect[key].sort
+	end
+
+	# we also could do this sort here...
+	collect.keys.each do |x|
+		collect[x] = collect[x].sort
+	end
+
+	final = []	# final order...
+	# TODO: here we can probably detect if this is an asymmetrical configurations, or maybe bad naming...
+	while collect.size > 0
+		collect.keys.sort.each do |x|
+
+			# NOTE: this array should already be sorted!
+			p = collect[x].shift	# assume an array of at least 1 element
+			final.push( { 'host' => x, 'path' => p } )	# save
+
+			if collect[x].size == 0		# maybe the array is empty now
+				collect.delete(x)	# remove that empty list's key
+			end
+
+		end
+	end
+
+	# build final result
+	result[group] = final.collect {|x| x['host']+':'+x['path'] }
+end
+
+# build the correctly sorted brick list...
+result.keys.each do |x|
+	Facter.add('gluster_brick_group_'+x) do
+		#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+		setcode {
+			# don't reuse single variable to avoid bug #:
+			# http://projects.puppetlabs.com/issues/22455
+			# TODO: facter should support native list types :)
+			result[x].join(',')
+		}
+	end
+end
+
+# list of generated gluster_ports_volume's
+Facter.add('gluster_brick_group_facts') do
+	#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+	setcode {
+		result.keys.collect {|x| 'gluster_brick_group_'+x }.join(',')
+	}
+end
+
+# vim: ts=8
diff --git a/lib/facter/gluster_fsm.rb b/lib/facter/gluster_fsm.rb
new file mode 100644
index 0000000..887e869
--- /dev/null
+++ b/lib/facter/gluster_fsm.rb
@@ -0,0 +1,162 @@
+# GlusterFS module by James
+# Copyright (C) 2010-2013+ James Shubin
+# Written by James Shubin <james@shubin.ca>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'facter'
+require 'base64'	# TODO: i wish facter worked with types instead of hacks
+
+# regexp to match gluster volume name, eg: testvolume
+volume_regexp = /^[a-z]{1}[a-z0-9]{0,}$/	# TODO: is this perfect ?
+
+# returns true if each brick in the list matches
+def brick_match(l)
+	# regexp to match a brick pattern, eg: annex1.example.com:/storage1a
+	brick_regexp = /^[a-zA-Z]{1}[a-zA-Z0-9\.\-]{0,}:\/[a-zA-Z0-9]{1}[a-zA-Z0-9\/\.\-]{0,}$/	# TODO: is this perfect ?
+	l.each do |x|
+		if not brick_regexp.match(x)
+			return false
+		end
+	end
+	return true
+end
+
+# find the module_vardir
+dir = Facter.value('puppet_vardirtmp')		# nil if missing
+if dir.nil?					# let puppet decide if present!
+	dir = Facter.value('puppet_vardir')
+	if dir.nil?
+		var = nil
+	else
+		var = dir.gsub(/\/$/, '')+'/'+'tmp/'	# ensure trailing slash
+	end
+else
+	var = dir.gsub(/\/$/, '')+'/'
+end
+
+if var.nil?
+	# if we can't get a valid vardirtmp, then we can't collect...
+	valid_dir = nil
+else
+	module_vardir = var+'gluster/'
+	valid_dir = module_vardir.gsub(/\/$/, '')+'/'
+	fsm_dir = valid_dir+'volume/fsm/'
+end
+
+state = {}
+stack = {}
+watch = {}
+
+if not(fsm_dir.nil?) and File.directory?(fsm_dir)
+	# loop through each sub directory in the gluster::volume fsm
+	Dir.glob(fsm_dir+'*').each do |d|
+		n = File.basename(d)	# should be the gluster::volume name
+		if n.length > 0 and volume_regexp.match(n)
+
+			f = d.gsub(/\/$/, '')+'/state'	# full file path
+			if File.exists?(f)
+				# TODO: future versions should unpickle (but with yaml)
+				v = File.open(f, 'r').read.strip	# read into str
+				if v.length > 0 and brick_match(v.split(','))
+					state[n] = v
+				end
+			end
+
+			f = d.gsub(/\/$/, '')+'/stack'	# full file path
+			if File.exists?(f)
+				stack[n] = []	# initialize empty array
+				File.readlines(f).each do |l|
+					l = l.strip	# clean off /n's
+					# TODO: future versions should unpickle (but with yaml)
+					if l.length > 0 and brick_match(l.split(','))
+						#stack[n].push(l)
+						stack[n].push(Base64.encode64(l).delete("\n"))
+					end
+				end
+			end
+
+			f = d.gsub(/\/$/, '')+'/watch'	# full file path
+			if File.exists?(f)
+				watch[n] = []	# initialize empty array
+				File.readlines(f).each do |l|
+					l = l.strip	# clean off /n's
+					# TODO: future versions should unpickle (but with yaml)
+					if l.length > 0 and brick_match(l.split(','))
+						#watch[n].push(l)
+						watch[n].push(Base64.encode64(l).delete("\n"))
+					end
+				end
+			end
+
+		end
+	end
+end
+
+state.keys.each do |x|
+	Facter.add('gluster_volume_fsm_state_'+x) do
+		#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+		setcode {
+			state[x]
+		}
+	end
+
+	if stack.key?(x)
+		Facter.add('gluster_volume_fsm_stack_'+x) do
+			#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+			setcode {
+				stack[x].join(',')
+			}
+		end
+	end
+
+	if watch.key?(x)
+		Facter.add('gluster_volume_fsm_watch_'+x) do
+			#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+			setcode {
+				watch[x].join(',')
+			}
+		end
+	end
+end
+
+# list of gluster volume fsm state fact names
+Facter.add('gluster_volume_fsm_states') do
+	#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+	setcode {
+		state.keys.sort.collect {|x| 'gluster_volume_fsm_state_'+x }.join(',')
+	}
+end
+
+Facter.add('gluster_volume_fsm_stacks') do
+	#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+	setcode {
+		(state.keys & stack.keys).sort.collect {|x| 'gluster_volume_fsm_stack_'+x }.join(',')
+	}
+end
+
+Facter.add('gluster_volume_fsm_watchs') do
+	#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+	setcode {
+		(state.keys & watch.keys).sort.collect {|x| 'gluster_volume_fsm_watch_'+x }.join(',')
+	}
+end
+
+Facter.add('gluster_fsm_debug') do
+	#confine :operatingsystem => %w{CentOS, RedHat, Fedora}
+	setcode {
+		'Oh cool, james added fsm support to puppet-gluster. Sweet!'
+	}
+end
+
diff --git a/manifests/brick.pp b/manifests/brick.pp
index fe13f5d..8ab9241 100644
--- a/manifests/brick.pp
+++ b/manifests/brick.pp
@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 define gluster::brick(
+	$group = 'default',		# grouping for multiple puppet-glusters
 	# if dev is false, path in $name is used directly after a mkdir -p
 	$dev = false,			# /dev/sdc, /dev/disk/by-id/scsi-36003048007e14f0014ca2743150a5471
 	$fsuuid = '',			# set a uuid for this fs (uuidgen)
@@ -27,6 +28,12 @@ define gluster::brick(
 	$force = false,			# if true, this will overwrite any xfs fs it sees, useful for rebuilding gluster and wiping data. NOTE: there are other safeties in place to stop this.
 	$areyousure = false		# do you allow puppet to do dangerous things ?
 ) {
+	include gluster::brick::base
+	include gluster::vardir
+
+	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
+	$vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '')
+
 	# eg: annex1.example.com:/storage1a
 	$split = split($name, ':')	# do some $name parsing
 	$host = $split[0]		# host fqdn
@@ -44,6 +51,17 @@ define gluster::brick(
 
 	Gluster::Host[$host] -> Gluster::Brick[$name]	# brick requires host
 
+	# create a brick tag to be collected by the gluster_brick_group_* fact!
+	$safename = regsubst("${name}", '/', '_', 'G')	# make /'s safe
+	file { "${vardir}/brick/${safename}.${group}":
+		content => "${name}\n",
+		owner => root,
+		group => root,
+		mode => 644,
+		ensure => present,
+		require => File["${vardir}/brick/"],
+	}
+
 	$ro_bool = $ro ? {		# this has been added as a convenience
 		true => 'ro',
 		default => 'rw',
diff --git a/manifests/brick/base.pp b/manifests/brick/base.pp
new file mode 100644
index 0000000..83398e2
--- /dev/null
+++ b/manifests/brick/base.pp
@@ -0,0 +1,32 @@
+# GlusterFS module by James
+# Copyright (C) 2010-2013+ James Shubin
+# Written by James Shubin <james@shubin.ca>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+class gluster::brick::base {
+
+	include gluster::vardir
+	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
+	$vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '')
+
+	file { "${vardir}/brick/":
+		ensure => directory,	# make sure this is a directory
+		recurse => true,	# don't recurse into directory
+		purge => true,		# don't purge unmanaged files
+		force => true,		# don't purge subdirs and links
+		require => File["${vardir}/"],
+	}
+}
+# vim: ts=8
diff --git a/manifests/simple.pp b/manifests/simple.pp
new file mode 100644
index 0000000..4f34cb9
--- /dev/null
+++ b/manifests/simple.pp
@@ -0,0 +1,89 @@
+# GlusterFS module by James
+# Copyright (C) 2010-2013+ James Shubin
+# Written by James Shubin <james@shubin.ca>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+class gluster::simple(
+	$path = '',
+	$volume = 'puppet',	# NOTE: this can be a list...
+	$replica = 1,
+	$stripe = 1,		# TODO: not fully implemented in puppet-gluster
+	$vip = ''		# strongly recommended
+) {
+	include gluster::vardir
+
+	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
+	$vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '')
+
+	if "${path}" == '' {
+		file { "${vardir}/data/":
+			ensure => directory,	# make sure this is a directory
+			recurse => false,	# don't recurse into directory
+			purge => false,		# don't purge unmanaged files
+			force => false,		# don't purge subdirs and links
+			require => File["${vardir}/"],
+		}
+	}
+
+	$chosen_path = "${path}" ? {
+		'' => "${vardir}/data/",
+		default => "${path}",
+	}
+
+	$valid_path = sprintf("%s/", regsubst($chosen_path, '\/$', ''))
+
+	notify { 'gluster::simple':
+		message => 'You are using gluster::simple !',
+	}
+
+	if "${vip}" == '' {
+		# If you don't use a VIP, things will be racy, but could mostly
+		# work. If you run puppet manually, then a vip isn't necessary.
+		# see: http://ttboj.wordpress.com/2012/08/23/how-to-avoid-cluster-race-conditions-or-how-to-implement-a-distributed-lock-manager-in-puppet/
+		warning('It is highly recommended to use a VIP.')
+	}
+
+	class { '::gluster::server':
+		vip => "${vip}",
+		#zone => 'net',	# defaults to net
+		shorewall => true,
+	}
+
+	@@gluster::host { "${::fqdn}":
+	}
+	Gluster::Host <<||>>
+
+	@@gluster::brick { "${::fqdn}:${valid_path}":
+		areyousure => true,
+	}
+
+	Gluster::Brick <<||>>
+
+	gluster::volume { $volume:
+		replica => $replica,
+		stripe => $stripe,
+		# NOTE: with this method you do not choose the order of course!
+		# the gluster_fqdns fact is alphabetical, but not complete till
+		# at least a puppet run of each node has occured. watch out for
+		# partial clusters missing some of the nodes with bad ordering!
+		#bricks => split(inline_template("<%= gluster_fqdns.split(',').collect {|x| x+':${valid_path}' }.join(',') %>"), ','),
+		# the only semi-safe way is the new built in automatic collect:
+		bricks => true,			# automatic brick collection...
+		start => true,
+	}
+	Gluster::Volume <<||>>
+}
+
+# vim: ts=8
diff --git a/manifests/volume.pp b/manifests/volume.pp
index 8c79dd8..5e8b7e1 100644
--- a/manifests/volume.pp
+++ b/manifests/volume.pp
@@ -16,12 +16,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 define gluster::volume(
-	$bricks = [],
+	$bricks = true,		# specify a list of bricks, or true for auto...
+	$group = 'default',	# use this bricks group name if we auto collect
 	$transport = 'tcp',
 	$replica = 1,
 	$stripe = 1,
 	$vip = '',		# vip of the cluster (optional but recommended)
 	$ping = true,		# do we want to include fping checks ?
+	$settle = true,		# do we want to run settle checks ?
 	$start = undef		# start volume ? true, false (stop it) or undef
 ) {
 	include gluster::xml
@@ -34,6 +36,95 @@ define gluster::volume(
 	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
 	$vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '')
 
+	$settle_count = 3		# three is a reasonable default!
+	$maxlength = 3
+	if $maxlength < $settle_count {
+		fail('The $maxlength needs to be greater than or equal to the $settle_count.')
+	}
+	$are_bricks_collected = (type($bricks) == 'boolean' and ($bricks == true))
+	# NOTE: settle checks are still useful even if we are using ping checks
+	# the reason why they are still useful, is that they can detect changes
+	# in the bricks, which might propagate slowly because of exported types
+	# the fping checks can only verify that the individual hosts are alive!
+	$settle_count_check = $are_bricks_collected ? {
+		false => false,
+		default => $settle ? {
+			false => false,
+			default => true,
+		}
+	}
+	# TODO: implement settle_time_check
+	$settle_time_check = $settle ? {
+		false => false,
+		default => true,
+	}
+
+	# clean up old fsm data when not in use, because parent $vardir purges!
+	if $are_bricks_collected {
+		include gluster::volume::fsm
+		file { "${vardir}/volume/fsm/${name}/":
+			ensure => directory,	# make sure this is a directory
+			recurse => true,	# recurse into directory
+			purge => false,		# don't purge unmanaged files
+			force => false,		# don't purge subdirs and links
+			require => File["${vardir}/volume/fsm/"],
+		}
+	}
+
+	$collected_bricks = split(getvar("gluster_brick_group_${group}"), ',')
+	$valid_bricks = type($bricks) ? {
+		'boolean' => $bricks ? {
+			true => $collected_bricks,		# an array...
+			default => [],				# invalid type
+		},
+		'list' => $bricks,
+		default => [],					# invalid type
+	}
+
+	# helpful debugging!
+	notice(inline_template('valid_bricks: <%= valid_bricks.inspect %>'))
+
+	# NOTE: we're using the valid_bricks value here, and not the collected
+	# value. while we only need the $collected value for settle detection,
+	# the actual brick value is needed for future add/remove brick code...
+	$valid_input = join($valid_bricks, ',')	# TODO: this should be pickled
+	if $are_bricks_collected and "${valid_input}" == '' {
+		notice('The gluster::brick collection is not ready yet.')
+	}
+	$last = getvar("gluster_volume_fsm_state_${name}")	# fact !
+	$valid_last = "${last}" ? {
+		# initialize the $last var to match the $input if it's empty...
+		'' => "${valid_input}",
+		default => "${last}",
+	}
+	if $are_bricks_collected and ("${valid_input}" != '') and ("${valid_last}" == '') {
+		fail('Previous state is invalid.')	# fact was tampered with
+	}
+
+	# NOTE: the stack lists are left base64 encoded since we only care if they change! :P
+	# NOTE: each element in stack is base64 encoded because they contain: ,
+	$stack_fact = getvar("gluster_volume_fsm_stack_${name}")	# fact !
+	$stack_full = split("${stack_fact}", ',')
+	$stack_trim = "${maxlength}" ? {
+		'-1' => $stack_full,	# unlimited
+		#default => split(inline_template('<%= stack_full[0,maxlength.to_i.abs].join(",") %>'), ','),
+		default => split(inline_template('<%= stack_full[[stack_full.size-maxlength.to_i.abs,0].max,maxlength.to_i.abs].join(",") %>'), ','),
+	}
+
+	$watch_fact = getvar("gluster_volume_fsm_watch_${name}")	# fact !
+	$watch_full = split("${watch_fact}", ',')
+	$watch_trim = "${maxlength}" ? {
+		'-1' => $watch_full,	# unlimited
+		#default => split(inline_template('<%= watch_full[0,maxlength.to_i.abs].join(",") %>'), ','),
+		default => split(inline_template('<%= watch_full[[watch_full.size-maxlength.to_i.abs,0].max,maxlength.to_i.abs].join(",") %>'), ','),
+	}
+
+	# if the last $settle_count elements are the same, the template
+	# should reduce down to the value '1'. check this and the size.
+	$one = inline_template('<%= watch_trim[[watch_trim.size-settle_count.to_i,0].max,settle_count.to_i].uniq.size %>')
+	$watch_trim_size = size($watch_trim)
+	$settled = ((! $settle_count_check) or ((size($watch_trim) >= $settle_count) and "${one}" == '1'))
+
 	# TODO: if using rdma, maybe we should pull in the rdma package... ?
 	$valid_transport = $transport ? {
 		'rdma' => 'rdma',
@@ -59,24 +150,24 @@ define gluster::volume(
 	# returns interface name that has vip, or '' if none are found.
 	$vipif = inline_template("<%= interfaces.split(',').find_all {|x| '${valid_vip}' == scope.lookupvar('ipaddress_'+x) }[0,1].join('') %>")
 
-	#Gluster::Brick[$bricks] -> Gluster::Volume[$name]	# volume requires bricks
+	#Gluster::Brick[$valid_bricks] -> Gluster::Volume[$name]	# volume requires bricks
 
 	# get the bricks that match our fqdn, and append /$name to their path.
 	# return only these paths, which can be used to build the volume dirs.
 	# NOTE: gluster v3.4 won't create a volume if this dir already exists.
 	# TODO: is this needed when bricks are devices and not on filesystem ?
-	#$volume_dirs = split(inline_template("<%= bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}' }.join(' ') %>"), ' ')
+	#$volume_dirs = split(inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}' }.join(' ') %>"), ' ')
 	#file { $volume_dirs:
 	#	ensure => directory,		# make sure this is a directory
 	#	recurse => false,		# don't recurse into directory
 	#	purge => false,			# don't purge unmanaged files
 	#	force => false,			# don't purge subdirs and links
 	#	before => Exec["gluster-volume-create-${name}"],
-	#	require => Gluster::Brick[$bricks],
+	#	require => Gluster::Brick[$valid_bricks],
 	#}
 
 	# add /${name} to the end of each: brick:/path entry
-	$brick_spec = inline_template("<%= bricks.collect {|x| ''+x.chomp('/')+'/${name}' }.join(' ') %>")
+	$brick_spec = inline_template("<%= valid_bricks.collect {|x| ''+x.chomp('/')+'/${name}' }.join(' ') %>")
 
 	# if volume creation fails for a stupid reason, in many cases, glusterd
 	# already did some of the work and left us with volume name directories
@@ -85,10 +176,10 @@ define gluster::volume(
 	# we error we should rmdir any empty volume dirs to keep it pristine...
 	# TODO: this should be a gluster bug... we must hope it doesn't happen!
 	# maybe related to: https://bugzilla.redhat.com/show_bug.cgi?id=835494
-	$rmdir_volume_dirs = sprintf("/bin/rmdir '%s'", inline_template("<%= bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}/' }.join('\' \'') %>"))
+	$rmdir_volume_dirs = sprintf("/bin/rmdir '%s'", inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}/' }.join('\' \'') %>"))
 
 	# get the list of bricks fqdn's that don't have our fqdn
-	$others = inline_template("<%= bricks.find_all{|x| x.split(':')[0] != '${fqdn}' }.collect {|y| y.split(':')[0] }.join(' ') %>")
+	$others = inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] != '${fqdn}' }.collect {|y| y.split(':')[0] }.join(' ') %>")
 
 	$fping = sprintf("/usr/sbin/fping -q %s", $others)
 	$status = sprintf("/usr/sbin/gluster peer status --xml | ${vardir}/xml.py connected %s", $others)
@@ -106,14 +197,14 @@ define gluster::volume(
 			Service['glusterd'],
 			File["${vardir}/volume/create-${name}.sh"],
 			File["${vardir}/xml.py"],	# status check
-			Gluster::Brick[$bricks],
+			Gluster::Brick[$valid_bricks],
 		],
 		default => [
 			Service['glusterd'],
 			File["${vardir}/volume/create-${name}.sh"],
 			Package['fping'],
 			File["${vardir}/xml.py"],	# status check
-			Gluster::Brick[$bricks],
+			Gluster::Brick[$valid_bricks],
 		],
 	}
 
@@ -145,14 +236,17 @@ define gluster::volume(
 		# NOTE: in this command, we save the std{out,err} and pass them
 		# on too for puppet to consume. we save in /tmp for fast access
 		# EXAMPLE: gluster volume create test replica 2 transport tcp annex1.example.com:/storage1a/test annex2.example.com:/storage2a/test annex3.example.com:/storage3b/test annex4.example.com:/storage4b/test annex1.example.com:/storage1c/test annex2.example.com:/storage2c/test annex3.example.com:/storage3d/test annex4.example.com:/storage4d/test
-		exec { "gluster-volume-create-${name}":
-			command => "${vardir}/volume/create-${name}.sh",
-			logoutput => on_failure,
-			unless => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -",	# add volume if it doesn't exist
-			onlyif => $onlyif,
-			#before => TODO?,
-			require => $require,
-			alias => "gluster-volume-create-${name}",
+		# TODO: add a timer similar to my puppet-runonce timer code... to wait X minutes after we've settled... useful if we Exec['again'] to speed things up...
+		if $settled {
+			exec { "gluster-volume-create-${name}":
+				command => "${vardir}/volume/create-${name}.sh",
+				logoutput => on_failure,
+				unless => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -",	# add volume if it doesn't exist
+				onlyif => $onlyif,
+				#before => TODO?,
+				require => $require,
+				alias => "gluster-volume-create-${name}",
+			}
 		}
 	}
 
@@ -164,7 +258,10 @@ define gluster::volume(
 				logoutput => on_failure,
 				onlyif => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -",
 				unless => "/usr/sbin/gluster volume status ${name}",	# returns false if stopped
-				require => Exec["gluster-volume-create-${name}"],
+				require => $settled ? {	# require if type exists
+					false => undef,
+					default => Exec["gluster-volume-create-${name}"],
+				},
 				alias => "gluster-volume-start-${name}",
 			}
 		} elsif ( $start == false ) {
@@ -177,7 +274,10 @@ define gluster::volume(
 			exec { "/usr/bin/yes | /usr/sbin/gluster volume stop ${name}":
 				logoutput => on_failure,
 				onlyif => "/usr/sbin/gluster volume status ${name}",	# returns true if started
-				require => Exec["gluster-volume-create-${name}"],
+				require => $settled ? {	# require if type exists
+					false => undef,
+					default => Exec["gluster-volume-create-${name}"],
+				},
 				alias => "gluster-volume-stop-${name}",
 			}
 		} else {	# 'undef'-ined
@@ -218,6 +318,61 @@ define gluster::volume(
 			source => "${zone}",	# use our source zone
 			before => Service['glusterd'],
 		}
+
+	# fsm variables and boilerplate
+	$statefile = "${vardir}/volume/fsm/${name}/state"
+	$stackfile = "${vardir}/volume/fsm/${name}/stack"
+	$watchfile = "${vardir}/volume/fsm/${name}/watch"
+	$diff = "/usr/bin/test '${valid_input}' != '${valid_last}'"
+	$stack_truncate = "${maxlength}" ? {
+		'-1' => '',	# unlimited
+		#default => sprintf("&& /bin/sed -i '%d,$ d' ${stackfile}", inline_template('<%= maxlength.to_i.abs+1 %>')),
+		default => sprintf(" && (/bin/grep -v '^$' ${stackfile} | /usr/bin/tail -%d | /usr/bin/tee ${stackfile})", inline_template('<%= maxlength.to_i.abs %>')),
+	}
+	$watch_truncate = "${maxlength}" ? {
+		'-1' => '',	# unlimited
+		#default => sprintf("&& /bin/sed -i '%d,$ d' ${watchfile}", inline_template('<%= maxlength.to_i.abs+1 %>')),
+		default => sprintf(" && (/bin/grep -v '^$' ${watchfile} | /usr/bin/tail -%d | /usr/bin/tee ${watchfile})", inline_template('<%= maxlength.to_i.abs %>')),
+	}
+
+	if $are_bricks_collected and ("${valid_input}" != '') {	# ready or not?
+
+		# TODO: future versions should pickle (but with yaml)
+		exec { "/bin/echo '${valid_input}' > '${statefile}'":
+			logoutput => on_failure,
+			onlyif => "/usr/bin/test ! -e '${statefile}' || ${diff}",
+			require => File["${vardir}/volume/fsm/${name}/"],
+			alias => "gluster-volume-fsm-state-${name}",
+		}
+
+		# NOTE: keep a stack of past transitions, and load them in as a list...
+		exec { "/bin/echo '${valid_input}' >> '${stackfile}'${stack_truncate}":
+			logoutput => on_failure,
+			onlyif => "/usr/bin/test ! -e '${stackfile}' || ${diff}",
+			require => [
+				File["${vardir}/volume/fsm/${name}/"],
+				# easy way to ensure the transition types don't need to
+				# add a before to both exec's since this one follows it
+				Exec["gluster-volume-fsm-state-${name}"],
+			],
+			alias => "gluster-volume-fsm-stack-${name}",
+		}
+
+		# NOTE: watch *all* transitions, and load them in as a list...
+		exec { "/bin/echo '${valid_input}' >> '${watchfile}'${watch_truncate}":
+			logoutput => on_failure,
+			# we run this if the file doesn't exist, or there is a
+			# difference to record, or the sequence hasn't settled
+			# we also check that we have our minimum settle count!
+			onlyif => "/usr/bin/test ! -e '${watchfile}' || ${diff} || /usr/bin/test '1' != '${one}' || /usr/bin/test ${watch_trim_size} -lt ${settle_count}",
+			require => [
+				File["${vardir}/volume/fsm/${name}/"],
+				# easy way to ensure the transition types don't need to
+				# add a before to both exec's since this one follows it
+				Exec["gluster-volume-fsm-state-${name}"],
+			],
+			alias => "gluster-volume-fsm-watch-${name}",
+		}
 	}
 }
 
diff --git a/manifests/volume/base.pp b/manifests/volume/base.pp
index f284a16..1991ce7 100644
--- a/manifests/volume/base.pp
+++ b/manifests/volume/base.pp
@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-class gluster::volume::base {	
+class gluster::volume::base {
 
 	include gluster::vardir
 	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
diff --git a/manifests/volume/fsm.pp b/manifests/volume/fsm.pp
new file mode 100644
index 0000000..c780ede
--- /dev/null
+++ b/manifests/volume/fsm.pp
@@ -0,0 +1,32 @@
+# GlusterFS module by James
+# Copyright (C) 2010-2013+ James Shubin
+# Written by James Shubin <james@shubin.ca>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+class gluster::volume::fsm {
+
+	include gluster::vardir
+	#$vardir = $::gluster::vardir::module_vardir	# with trailing slash
+	$vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '')
+
+	file { "${vardir}/volume/fsm/":
+		ensure => directory,	# make sure this is a directory
+		recurse => true,	# don't recurse into directory
+		purge => true,		# don't purge unmanaged files
+		force => true,		# don't purge subdirs and links
+		require => File["${vardir}/volume/"],
+	}
+}
+# vim: ts=8