diff options
Diffstat (limited to 'manifests')
| -rw-r--r-- | manifests/brick.pp | 18 | ||||
| -rw-r--r-- | manifests/brick/base.pp | 32 | ||||
| -rw-r--r-- | manifests/simple.pp | 89 | ||||
| -rw-r--r-- | manifests/volume.pp | 193 | ||||
| -rw-r--r-- | manifests/volume/base.pp | 2 | ||||
| -rw-r--r-- | manifests/volume/fsm.pp | 32 |
6 files changed, 346 insertions, 20 deletions
diff --git a/manifests/brick.pp b/manifests/brick.pp index fe13f5d..8ab9241 100644 --- a/manifests/brick.pp +++ b/manifests/brick.pp @@ -16,6 +16,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. define gluster::brick( + $group = 'default', # grouping for multiple puppet-glusters # if dev is false, path in $name is used directly after a mkdir -p $dev = false, # /dev/sdc, /dev/disk/by-id/scsi-36003048007e14f0014ca2743150a5471 $fsuuid = '', # set a uuid for this fs (uuidgen) @@ -27,6 +28,12 @@ define gluster::brick( $force = false, # if true, this will overwrite any xfs fs it sees, useful for rebuilding gluster and wiping data. NOTE: there are other safeties in place to stop this. $areyousure = false # do you allow puppet to do dangerous things ? ) { + include gluster::brick::base + include gluster::vardir + + #$vardir = $::gluster::vardir::module_vardir # with trailing slash + $vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '') + # eg: annex1.example.com:/storage1a $split = split($name, ':') # do some $name parsing $host = $split[0] # host fqdn @@ -44,6 +51,17 @@ define gluster::brick( Gluster::Host[$host] -> Gluster::Brick[$name] # brick requires host + # create a brick tag to be collected by the gluster_brick_group_* fact! + $safename = regsubst("${name}", '/', '_', 'G') # make /'s safe + file { "${vardir}/brick/${safename}.${group}": + content => "${name}\n", + owner => root, + group => root, + mode => 644, + ensure => present, + require => File["${vardir}/brick/"], + } + $ro_bool = $ro ? { # this has been added as a convenience true => 'ro', default => 'rw', diff --git a/manifests/brick/base.pp b/manifests/brick/base.pp new file mode 100644 index 0000000..83398e2 --- /dev/null +++ b/manifests/brick/base.pp @@ -0,0 +1,32 @@ +# GlusterFS module by James +# Copyright (C) 2010-2013+ James Shubin +# Written by James Shubin <james@shubin.ca> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +class gluster::brick::base { + + include gluster::vardir + #$vardir = $::gluster::vardir::module_vardir # with trailing slash + $vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '') + + file { "${vardir}/brick/": + ensure => directory, # make sure this is a directory + recurse => true, # don't recurse into directory + purge => true, # don't purge unmanaged files + force => true, # don't purge subdirs and links + require => File["${vardir}/"], + } +} +# vim: ts=8 diff --git a/manifests/simple.pp b/manifests/simple.pp new file mode 100644 index 0000000..4f34cb9 --- /dev/null +++ b/manifests/simple.pp @@ -0,0 +1,89 @@ +# GlusterFS module by James +# Copyright (C) 2010-2013+ James Shubin +# Written by James Shubin <james@shubin.ca> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +class gluster::simple( + $path = '', + $volume = 'puppet', # NOTE: this can be a list... + $replica = 1, + $stripe = 1, # TODO: not fully implemented in puppet-gluster + $vip = '' # strongly recommended +) { + include gluster::vardir + + #$vardir = $::gluster::vardir::module_vardir # with trailing slash + $vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '') + + if "${path}" == '' { + file { "${vardir}/data/": + ensure => directory, # make sure this is a directory + recurse => false, # don't recurse into directory + purge => false, # don't purge unmanaged files + force => false, # don't purge subdirs and links + require => File["${vardir}/"], + } + } + + $chosen_path = "${path}" ? { + '' => "${vardir}/data/", + default => "${path}", + } + + $valid_path = sprintf("%s/", regsubst($chosen_path, '\/$', '')) + + notify { 'gluster::simple': + message => 'You are using gluster::simple !', + } + + if "${vip}" == '' { + # If you don't use a VIP, things will be racy, but could mostly + # work. If you run puppet manually, then a vip isn't necessary. + # see: http://ttboj.wordpress.com/2012/08/23/how-to-avoid-cluster-race-conditions-or-how-to-implement-a-distributed-lock-manager-in-puppet/ + warning('It is highly recommended to use a VIP.') + } + + class { '::gluster::server': + vip => "${vip}", + #zone => 'net', # defaults to net + shorewall => true, + } + + @@gluster::host { "${::fqdn}": + } + Gluster::Host <<||>> + + @@gluster::brick { "${::fqdn}:${valid_path}": + areyousure => true, + } + + Gluster::Brick <<||>> + + gluster::volume { $volume: + replica => $replica, + stripe => $stripe, + # NOTE: with this method you do not choose the order of course! + # the gluster_fqdns fact is alphabetical, but not complete till + # at least a puppet run of each node has occured. watch out for + # partial clusters missing some of the nodes with bad ordering! + #bricks => split(inline_template("<%= gluster_fqdns.split(',').collect {|x| x+':${valid_path}' }.join(',') %>"), ','), + # the only semi-safe way is the new built in automatic collect: + bricks => true, # automatic brick collection... + start => true, + } + Gluster::Volume <<||>> +} + +# vim: ts=8 diff --git a/manifests/volume.pp b/manifests/volume.pp index 8c79dd8..5e8b7e1 100644 --- a/manifests/volume.pp +++ b/manifests/volume.pp @@ -16,12 +16,14 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. define gluster::volume( - $bricks = [], + $bricks = true, # specify a list of bricks, or true for auto... + $group = 'default', # use this bricks group name if we auto collect $transport = 'tcp', $replica = 1, $stripe = 1, $vip = '', # vip of the cluster (optional but recommended) $ping = true, # do we want to include fping checks ? + $settle = true, # do we want to run settle checks ? $start = undef # start volume ? true, false (stop it) or undef ) { include gluster::xml @@ -34,6 +36,95 @@ define gluster::volume( #$vardir = $::gluster::vardir::module_vardir # with trailing slash $vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '') + $settle_count = 3 # three is a reasonable default! + $maxlength = 3 + if $maxlength < $settle_count { + fail('The $maxlength needs to be greater than or equal to the $settle_count.') + } + $are_bricks_collected = (type($bricks) == 'boolean' and ($bricks == true)) + # NOTE: settle checks are still useful even if we are using ping checks + # the reason why they are still useful, is that they can detect changes + # in the bricks, which might propagate slowly because of exported types + # the fping checks can only verify that the individual hosts are alive! + $settle_count_check = $are_bricks_collected ? { + false => false, + default => $settle ? { + false => false, + default => true, + } + } + # TODO: implement settle_time_check + $settle_time_check = $settle ? { + false => false, + default => true, + } + + # clean up old fsm data when not in use, because parent $vardir purges! + if $are_bricks_collected { + include gluster::volume::fsm + file { "${vardir}/volume/fsm/${name}/": + ensure => directory, # make sure this is a directory + recurse => true, # recurse into directory + purge => false, # don't purge unmanaged files + force => false, # don't purge subdirs and links + require => File["${vardir}/volume/fsm/"], + } + } + + $collected_bricks = split(getvar("gluster_brick_group_${group}"), ',') + $valid_bricks = type($bricks) ? { + 'boolean' => $bricks ? { + true => $collected_bricks, # an array... + default => [], # invalid type + }, + 'list' => $bricks, + default => [], # invalid type + } + + # helpful debugging! + notice(inline_template('valid_bricks: <%= valid_bricks.inspect %>')) + + # NOTE: we're using the valid_bricks value here, and not the collected + # value. while we only need the $collected value for settle detection, + # the actual brick value is needed for future add/remove brick code... + $valid_input = join($valid_bricks, ',') # TODO: this should be pickled + if $are_bricks_collected and "${valid_input}" == '' { + notice('The gluster::brick collection is not ready yet.') + } + $last = getvar("gluster_volume_fsm_state_${name}") # fact ! + $valid_last = "${last}" ? { + # initialize the $last var to match the $input if it's empty... + '' => "${valid_input}", + default => "${last}", + } + if $are_bricks_collected and ("${valid_input}" != '') and ("${valid_last}" == '') { + fail('Previous state is invalid.') # fact was tampered with + } + + # NOTE: the stack lists are left base64 encoded since we only care if they change! :P + # NOTE: each element in stack is base64 encoded because they contain: , + $stack_fact = getvar("gluster_volume_fsm_stack_${name}") # fact ! + $stack_full = split("${stack_fact}", ',') + $stack_trim = "${maxlength}" ? { + '-1' => $stack_full, # unlimited + #default => split(inline_template('<%= stack_full[0,maxlength.to_i.abs].join(",") %>'), ','), + default => split(inline_template('<%= stack_full[[stack_full.size-maxlength.to_i.abs,0].max,maxlength.to_i.abs].join(",") %>'), ','), + } + + $watch_fact = getvar("gluster_volume_fsm_watch_${name}") # fact ! + $watch_full = split("${watch_fact}", ',') + $watch_trim = "${maxlength}" ? { + '-1' => $watch_full, # unlimited + #default => split(inline_template('<%= watch_full[0,maxlength.to_i.abs].join(",") %>'), ','), + default => split(inline_template('<%= watch_full[[watch_full.size-maxlength.to_i.abs,0].max,maxlength.to_i.abs].join(",") %>'), ','), + } + + # if the last $settle_count elements are the same, the template + # should reduce down to the value '1'. check this and the size. + $one = inline_template('<%= watch_trim[[watch_trim.size-settle_count.to_i,0].max,settle_count.to_i].uniq.size %>') + $watch_trim_size = size($watch_trim) + $settled = ((! $settle_count_check) or ((size($watch_trim) >= $settle_count) and "${one}" == '1')) + # TODO: if using rdma, maybe we should pull in the rdma package... ? $valid_transport = $transport ? { 'rdma' => 'rdma', @@ -59,24 +150,24 @@ define gluster::volume( # returns interface name that has vip, or '' if none are found. $vipif = inline_template("<%= interfaces.split(',').find_all {|x| '${valid_vip}' == scope.lookupvar('ipaddress_'+x) }[0,1].join('') %>") - #Gluster::Brick[$bricks] -> Gluster::Volume[$name] # volume requires bricks + #Gluster::Brick[$valid_bricks] -> Gluster::Volume[$name] # volume requires bricks # get the bricks that match our fqdn, and append /$name to their path. # return only these paths, which can be used to build the volume dirs. # NOTE: gluster v3.4 won't create a volume if this dir already exists. # TODO: is this needed when bricks are devices and not on filesystem ? - #$volume_dirs = split(inline_template("<%= bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}' }.join(' ') %>"), ' ') + #$volume_dirs = split(inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}' }.join(' ') %>"), ' ') #file { $volume_dirs: # ensure => directory, # make sure this is a directory # recurse => false, # don't recurse into directory # purge => false, # don't purge unmanaged files # force => false, # don't purge subdirs and links # before => Exec["gluster-volume-create-${name}"], - # require => Gluster::Brick[$bricks], + # require => Gluster::Brick[$valid_bricks], #} # add /${name} to the end of each: brick:/path entry - $brick_spec = inline_template("<%= bricks.collect {|x| ''+x.chomp('/')+'/${name}' }.join(' ') %>") + $brick_spec = inline_template("<%= valid_bricks.collect {|x| ''+x.chomp('/')+'/${name}' }.join(' ') %>") # if volume creation fails for a stupid reason, in many cases, glusterd # already did some of the work and left us with volume name directories @@ -85,10 +176,10 @@ define gluster::volume( # we error we should rmdir any empty volume dirs to keep it pristine... # TODO: this should be a gluster bug... we must hope it doesn't happen! # maybe related to: https://bugzilla.redhat.com/show_bug.cgi?id=835494 - $rmdir_volume_dirs = sprintf("/bin/rmdir '%s'", inline_template("<%= bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}/' }.join('\' \'') %>")) + $rmdir_volume_dirs = sprintf("/bin/rmdir '%s'", inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] == '${fqdn}' }.collect {|y| y.split(':')[1].chomp('/')+'/${name}/' }.join('\' \'') %>")) # get the list of bricks fqdn's that don't have our fqdn - $others = inline_template("<%= bricks.find_all{|x| x.split(':')[0] != '${fqdn}' }.collect {|y| y.split(':')[0] }.join(' ') %>") + $others = inline_template("<%= valid_bricks.find_all{|x| x.split(':')[0] != '${fqdn}' }.collect {|y| y.split(':')[0] }.join(' ') %>") $fping = sprintf("/usr/sbin/fping -q %s", $others) $status = sprintf("/usr/sbin/gluster peer status --xml | ${vardir}/xml.py connected %s", $others) @@ -106,14 +197,14 @@ define gluster::volume( Service['glusterd'], File["${vardir}/volume/create-${name}.sh"], File["${vardir}/xml.py"], # status check - Gluster::Brick[$bricks], + Gluster::Brick[$valid_bricks], ], default => [ Service['glusterd'], File["${vardir}/volume/create-${name}.sh"], Package['fping'], File["${vardir}/xml.py"], # status check - Gluster::Brick[$bricks], + Gluster::Brick[$valid_bricks], ], } @@ -145,14 +236,17 @@ define gluster::volume( # NOTE: in this command, we save the std{out,err} and pass them # on too for puppet to consume. we save in /tmp for fast access # EXAMPLE: gluster volume create test replica 2 transport tcp annex1.example.com:/storage1a/test annex2.example.com:/storage2a/test annex3.example.com:/storage3b/test annex4.example.com:/storage4b/test annex1.example.com:/storage1c/test annex2.example.com:/storage2c/test annex3.example.com:/storage3d/test annex4.example.com:/storage4d/test - exec { "gluster-volume-create-${name}": - command => "${vardir}/volume/create-${name}.sh", - logoutput => on_failure, - unless => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -", # add volume if it doesn't exist - onlyif => $onlyif, - #before => TODO?, - require => $require, - alias => "gluster-volume-create-${name}", + # TODO: add a timer similar to my puppet-runonce timer code... to wait X minutes after we've settled... useful if we Exec['again'] to speed things up... + if $settled { + exec { "gluster-volume-create-${name}": + command => "${vardir}/volume/create-${name}.sh", + logoutput => on_failure, + unless => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -", # add volume if it doesn't exist + onlyif => $onlyif, + #before => TODO?, + require => $require, + alias => "gluster-volume-create-${name}", + } } } @@ -164,7 +258,10 @@ define gluster::volume( logoutput => on_failure, onlyif => "/usr/sbin/gluster volume list | /bin/grep -qxF '${name}' -", unless => "/usr/sbin/gluster volume status ${name}", # returns false if stopped - require => Exec["gluster-volume-create-${name}"], + require => $settled ? { # require if type exists + false => undef, + default => Exec["gluster-volume-create-${name}"], + }, alias => "gluster-volume-start-${name}", } } elsif ( $start == false ) { @@ -177,7 +274,10 @@ define gluster::volume( exec { "/usr/bin/yes | /usr/sbin/gluster volume stop ${name}": logoutput => on_failure, onlyif => "/usr/sbin/gluster volume status ${name}", # returns true if started - require => Exec["gluster-volume-create-${name}"], + require => $settled ? { # require if type exists + false => undef, + default => Exec["gluster-volume-create-${name}"], + }, alias => "gluster-volume-stop-${name}", } } else { # 'undef'-ined @@ -218,6 +318,61 @@ define gluster::volume( source => "${zone}", # use our source zone before => Service['glusterd'], } + + # fsm variables and boilerplate + $statefile = "${vardir}/volume/fsm/${name}/state" + $stackfile = "${vardir}/volume/fsm/${name}/stack" + $watchfile = "${vardir}/volume/fsm/${name}/watch" + $diff = "/usr/bin/test '${valid_input}' != '${valid_last}'" + $stack_truncate = "${maxlength}" ? { + '-1' => '', # unlimited + #default => sprintf("&& /bin/sed -i '%d,$ d' ${stackfile}", inline_template('<%= maxlength.to_i.abs+1 %>')), + default => sprintf(" && (/bin/grep -v '^$' ${stackfile} | /usr/bin/tail -%d | /usr/bin/tee ${stackfile})", inline_template('<%= maxlength.to_i.abs %>')), + } + $watch_truncate = "${maxlength}" ? { + '-1' => '', # unlimited + #default => sprintf("&& /bin/sed -i '%d,$ d' ${watchfile}", inline_template('<%= maxlength.to_i.abs+1 %>')), + default => sprintf(" && (/bin/grep -v '^$' ${watchfile} | /usr/bin/tail -%d | /usr/bin/tee ${watchfile})", inline_template('<%= maxlength.to_i.abs %>')), + } + + if $are_bricks_collected and ("${valid_input}" != '') { # ready or not? + + # TODO: future versions should pickle (but with yaml) + exec { "/bin/echo '${valid_input}' > '${statefile}'": + logoutput => on_failure, + onlyif => "/usr/bin/test ! -e '${statefile}' || ${diff}", + require => File["${vardir}/volume/fsm/${name}/"], + alias => "gluster-volume-fsm-state-${name}", + } + + # NOTE: keep a stack of past transitions, and load them in as a list... + exec { "/bin/echo '${valid_input}' >> '${stackfile}'${stack_truncate}": + logoutput => on_failure, + onlyif => "/usr/bin/test ! -e '${stackfile}' || ${diff}", + require => [ + File["${vardir}/volume/fsm/${name}/"], + # easy way to ensure the transition types don't need to + # add a before to both exec's since this one follows it + Exec["gluster-volume-fsm-state-${name}"], + ], + alias => "gluster-volume-fsm-stack-${name}", + } + + # NOTE: watch *all* transitions, and load them in as a list... + exec { "/bin/echo '${valid_input}' >> '${watchfile}'${watch_truncate}": + logoutput => on_failure, + # we run this if the file doesn't exist, or there is a + # difference to record, or the sequence hasn't settled + # we also check that we have our minimum settle count! + onlyif => "/usr/bin/test ! -e '${watchfile}' || ${diff} || /usr/bin/test '1' != '${one}' || /usr/bin/test ${watch_trim_size} -lt ${settle_count}", + require => [ + File["${vardir}/volume/fsm/${name}/"], + # easy way to ensure the transition types don't need to + # add a before to both exec's since this one follows it + Exec["gluster-volume-fsm-state-${name}"], + ], + alias => "gluster-volume-fsm-watch-${name}", + } } } diff --git a/manifests/volume/base.pp b/manifests/volume/base.pp index f284a16..1991ce7 100644 --- a/manifests/volume/base.pp +++ b/manifests/volume/base.pp @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -class gluster::volume::base { +class gluster::volume::base { include gluster::vardir #$vardir = $::gluster::vardir::module_vardir # with trailing slash diff --git a/manifests/volume/fsm.pp b/manifests/volume/fsm.pp new file mode 100644 index 0000000..c780ede --- /dev/null +++ b/manifests/volume/fsm.pp @@ -0,0 +1,32 @@ +# GlusterFS module by James +# Copyright (C) 2010-2013+ James Shubin +# Written by James Shubin <james@shubin.ca> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +class gluster::volume::fsm { + + include gluster::vardir + #$vardir = $::gluster::vardir::module_vardir # with trailing slash + $vardir = regsubst($::gluster::vardir::module_vardir, '\/$', '') + + file { "${vardir}/volume/fsm/": + ensure => directory, # make sure this is a directory + recurse => true, # don't recurse into directory + purge => true, # don't purge unmanaged files + force => true, # don't purge subdirs and links + require => File["${vardir}/volume/"], + } +} +# vim: ts=8 |