diff options
| -rwxr-xr-x | lib/puppet/filebucket.rb | 273 | ||||
| -rw-r--r-- | lib/puppet/type.rb | 1 | ||||
| -rwxr-xr-x | lib/puppet/type/filebucket.rb | 43 | ||||
| -rw-r--r-- | test/bucket/tc_bucket.rb | 284 | ||||
| -rw-r--r-- | test/types/tc_file.rb | 8 | ||||
| -rwxr-xr-x | test/types/tc_filebucket.rb | 117 |
6 files changed, 715 insertions, 11 deletions
diff --git a/lib/puppet/filebucket.rb b/lib/puppet/filebucket.rb new file mode 100755 index 000000000..8a613359c --- /dev/null +++ b/lib/puppet/filebucket.rb @@ -0,0 +1,273 @@ +#!/usr/bin/ruby -w + +#-------------------- +# accept and serve files +# +# $Id$ + + +require 'webrick' +#require 'webrick/https' +require 'xmlrpc/server' +require 'xmlrpc/client' +#require 'webrick/httpstatus' +require 'facter' +require 'digest/md5' +require 'base64' + +class BucketError < RuntimeError; end + +module FileBucket + DEFAULTPORT = 8139 + # this doesn't work for relative paths + def FileBucket.mkdir(dir) + if FileTest.exist?(dir) + return false + else + tmp = dir.sub(/^\//,'') + path = [File::SEPARATOR] + tmp.split(File::SEPARATOR).each { |dir| + path.push dir + unless FileTest.exist?(File.join(path)) + Dir.mkdir(File.join(path)) + end + } + return true + end + end + + def FileBucket.paths(base,md5) + return [ + File.join(base, md5), + File.join(base, md5, "contents"), + File.join(base, md5, "paths") + ] + end + + #--------------------------------------------------------------- + class Bucket + def initialize(hash) + # build our AST + + if hash.include?(:ConflictCheck) + @conflictchk = hash[:ConflictCheck] + hash.delete(:ConflictCheck) + else + @conflictchk = true + end + + if hash.include?(:Path) + @bucket = hash[:Path] + hash.delete(:Path) + else + @bucket = "/var/puppet/filebucket" + end + + # XXX this should really be done using Puppet::Type instances... + FileBucket.mkdir(@bucket) + end + + # accept a file from a client + def addfile(string,path) + #puts "entering addfile" + contents = Base64.decode64(string) + #puts "string is decoded" + + md5 = Digest::MD5.hexdigest(contents) + #puts "md5 is made" + + bpath, bfile, pathpath = FileBucket.paths(@bucket,md5) + + # if it's a new directory... + if FileBucket.mkdir(bpath) + # ...then just create the file + #puts "creating file" + File.open(bfile, File::WRONLY|File::CREAT) { |of| + of.print contents + } + #puts "File is created" + else # if the dir already existed... + # ...we need to verify that the contents match the existing file + if @conflictchk + unless FileTest.exists?(bfile) + raise(BucketError, + "No file at %s for sum %s" % [bfile,md5], caller) + end + + curfile = "" + File.open(bfile) { |of| + curfile = of.read + } + + # if the contents don't match, then we've found a conflict + # unlikely, but quite bad + if curfile != contents + raise(BucketError, + "Got passed new contents for sum %s" % md5, caller) + end + end + #puts "Conflict check is done" + end + + # in either case, add the passed path to the list of paths + paths = nil + addpath = false + if FileTest.exists?(pathpath) + File.open(pathpath) { |of| + paths = of.readlines + } + + # unless our path is already there... + unless paths.include?(path) + addpath = true + end + else + addpath = true + end + #puts "Path is checked" + + # if it's a new file, or if our path isn't in the file yet, add it + if addpath + File.open(pathpath, File::WRONLY|File::CREAT|File::APPEND) { |of| + of.puts path + } + #puts "Path is added" + end + + return md5 + end + + def getfile(md5) + bpath, bfile, bpaths = FileBucket.paths(@bucket,md5) + + unless FileTest.exists?(bfile) + return false + end + + contents = nil + File.open(bfile) { |of| + contents = of.read + } + + return Base64.encode64(contents) + end + + private + + def on_init + @default_namespace = 'urn:filebucket-server' + add_method(self, 'addfile', 'string', 'path') + add_method(self, 'getfile', 'md5') + end + + def cert(filename) + OpenSSL::X509::Certificate.new(File.open(File.join(@dir, filename)) { |f| + f.read + }) + end + + def key(filename) + OpenSSL::PKey::RSA.new(File.open(File.join(@dir, filename)) { |f| + f.read + }) + end + + end + #--------------------------------------------------------------- + + class BucketWebserver < WEBrick::HTTPServer + def initialize(hash) + unless hash.include?(:Port) + hash[:Port] = FileBucket::DEFAULTPORT + end + servlet = XMLRPC::WEBrickServlet.new + @bucket = FileBucket::Bucket.new(hash) + #puts @bucket + servlet.add_handler("bucket",@bucket) + super + + self.mount("/RPC2", servlet) + end + end + + class BucketClient < XMLRPC::Client + @@methods = [ :addfile, :getfile ] + + @@methods.each { |method| + self.send(:define_method,method) { |*args| + begin + call("bucket.%s" % method.to_s,*args) + rescue => detail + #puts detail + end + } + } + + def initialize(hash) + hash[:Path] ||= "/RPC2" + hash[:Server] ||= "localhost" + hash[:Port] ||= FileBucket::DEFAULTPORT + super(hash[:Server],hash[:Path],hash[:Port]) + end + end + + class Dipper + def initialize(hash) + if hash.include?(:Server) + @bucket = FileBucket::BucketClient.new( + :Server => hash[:Server] + ) + elsif hash.include?(:Bucket) + @bucket = hash[:Bucket] + elsif hash.include?(:Path) + @bucket = FileBucket::Bucket.new( + :Path => hash[:Path] + ) + end + end + + def backup(file) + unless FileTest.exists?(file) + raise(BucketError, "File %s does not exist" % file, caller) + end + contents = File.open(file) { |of| of.read } + + string = Base64.encode64(contents) + #puts "string is created" + + sum = @bucket.addfile(string,file) + #puts "file %s is added" % file + return sum + end + + def restore(file,sum) + restore = true + if FileTest.exists?(file) + contents = File.open(file) { |of| of.read } + + cursum = Digest::MD5.hexdigest(contents) + + # if the checksum has changed... + # this might be extra effort + if cursum == sum + restore = false + end + end + + if restore + #puts "Restoring %s" % file + newcontents = Base64.decode64(@bucket.getfile(sum)) + newsum = Digest::MD5.hexdigest(newcontents) + File.open(file,File::WRONLY|File::TRUNC) { |of| + of.print(newcontents) + } + #puts "Done" + return newsum + else + return nil + end + + end + end + #--------------------------------------------------------------- +end diff --git a/lib/puppet/type.rb b/lib/puppet/type.rb index 2d64740c7..4f58a971c 100644 --- a/lib/puppet/type.rb +++ b/lib/puppet/type.rb @@ -1078,6 +1078,7 @@ end require 'puppet/type/service' require 'puppet/type/exec' +require 'puppet/type/filebucket' require 'puppet/type/pfile' require 'puppet/type/symlink' require 'puppet/type/package' diff --git a/lib/puppet/type/filebucket.rb b/lib/puppet/type/filebucket.rb index e1b390e4d..537f3983b 100755 --- a/lib/puppet/type/filebucket.rb +++ b/lib/puppet/type/filebucket.rb @@ -5,29 +5,58 @@ require 'puppet/filebucket' module Puppet class Type - class FileBucket < Type + class PFileBucket < Type attr_reader :bucket @states = [] @parameters = [ :name, - :path + :server, + :path, + :port ] @name = :filebucket @namevar = :name + @@buckets = {} + + def self.bucket(name) + @@buckets[name] + end + def initialize(hash) super - unless self[:path] - self[:path] = File.join(Puppet[:puppetroot], "bucket") + if @parameters.include?(:server) + @parameters[:port] ||= FileBucket::DEFAULTPORT + begin + @bucket = FileBucket::Dipper.new( + :Server => @parameters[:server], + :Port => @parameters[:port] + ) + rescue => detail + raise Puppet::Error.new( + "Could not create remote filebucket: %s" % detail + ) + end + else + @parameters[:path] ||= File.join( + Puppet[:puppetroot], "bucket" + ) + begin + @bucket = FileBucket::Dipper.new( + :Path => @parameters[:path] + ) + rescue => detail + raise Puppet::Error.new( + "Could not create local filebucket: %s" % detail + ) + end end - @bucket = FileBucket::Bucket.new( - :Bucket => self[:path] - ) + @@buckets[self.name] = @bucket end end # Puppet::Type::Service diff --git a/test/bucket/tc_bucket.rb b/test/bucket/tc_bucket.rb new file mode 100644 index 000000000..3aa045fc1 --- /dev/null +++ b/test/bucket/tc_bucket.rb @@ -0,0 +1,284 @@ +if __FILE__ == $0 + $:.unshift '../../lib' + $:.unshift '../../../../library/trunk/lib/' + $:.unshift '../../../../library/trunk/test/' + $puppetbase = "../.." + $debug = true +else + $debug = false +end + +require 'puppet/filebucket' +require 'test/unit' +require 'puppettest.rb' +require 'base64' + +# $Id$ + + +$external = true +if ARGV[1] and ARGV[1] == "external" + $external = true +else + # default to external + #$external = false +end +class TestBucket < Test::Unit::TestCase + def debug(string) + if $debug + puts([Time.now,string].join(" ")) + end + end + + def filelist + files = [] + #/etc/passwd /etc/syslog.conf /etc/hosts + %w{ + who /tmp/bigfile sh uname /etc/passwd /etc/syslog.conf /etc/hosts + }.each { |file| + if file =~ /^\// + if FileTest.exists?(file) + files.push file + end + else + begin + path = %x{which #{file}} + rescue => detail + #STDERR.puts "Could not search for binaries: %s" % detail + next + end + + if path != "" + files.push path.chomp + end + end + } + + return files + end + + def setup + @bucket = File::SEPARATOR + File.join("tmp","filebuckettesting") + end + + def teardown + system("rm -rf %s" % @bucket) + if defined? $pid + system("kill -9 #{$pid} 2>/dev/null") + end + end + + def test_localserver + files = filelist() + server =nil + assert_nothing_raised { + server = FileBucket::Bucket.new( + :Bucket => @bucket + ) + } + files.each { |file| + contents = File.open(file) { |of| of.read } + + md5 = nil + assert_nothing_raised { + #STDERR.puts("adding %s" % file) if $debug + md5 = server.addfile(Base64.encode64(contents),file) + } + newcontents = nil + assert_nothing_raised { + #STDERR.puts("getting %s" % file) if $debug + newcontents = Base64.decode64(server.getfile(md5)) + } + + assert( + contents == newcontents + ) + } + end + + def test_localboth + files = filelist() + + tmpdir = File.join(@bucket,"tmpfiledir") + FileBucket.mkdir(tmpdir) + + server = nil + client = nil + threads = [] + assert_nothing_raised { + server = FileBucket::Bucket.new( + :Bucket => @bucket + ) + } + + #sleep(30) + assert_nothing_raised { + client = FileBucket::Dipper.new( + :Bucket => server + ) + } + files.each { |file| + name = File.basename(file) + tmppath = File.join(tmpdir,name) + + # copy the files to our tmp directory so we can modify them... + #STDERR.puts("copying %s" % file) if $debug + File.open(tmppath,File::WRONLY|File::TRUNC|File::CREAT) { |wf| + File.open(file) { |rf| + wf.print(rf.read) + } + } + + assert(FileTest.exists?(tmppath)) + + osum = nil + tsum = nil + nsum = nil + assert_nothing_raised { + #STDERR.puts("backing up %s" % file) if $debug + osum = client.backup(file) + } + assert_nothing_raised { + #STDERR.puts("backing up %s" % tmppath) if $debug + tsum = client.backup(tmppath) + } + + assert(tsum == osum) + + File.open(tmppath,File::WRONLY|File::TRUNC) { |wf| + wf.print "This is some test text\n" + } + assert_nothing_raised { + #STDERR.puts("backing up %s" % tmppath) if $debug + nsum = client.backup(tmppath) + } + + assert(tsum != nsum) + + assert_nothing_raised { + #STDERR.puts("restoring %s" % tmppath) if $debug + nsum = client.restore(tmppath,tsum) + } + + contents = File.open(tmppath) { |rf| + #STDERR.puts("reading %s" % tmppath) if $debug + rf.read + } + csum = Digest::MD5.hexdigest(contents) + assert(tsum == csum) + } + end + + def test_webxmlmix + files = filelist() + + tmpdir = File.join(@bucket,"tmpfiledir") + FileBucket.mkdir(tmpdir) + + server = nil + client = nil + port = FileBucket::DEFAULTPORT + serverthread = nil + pid = nil + if $external + $pid = fork { + server = FileBucket::BucketWebserver.new( + :Bucket => @bucket, + :Port => port + ) + trap(:INT) { server.shutdown } + trap(:TERM) { server.shutdown } + server.start + } + sleep 3 + #puts "pid is %s" % pid + #exit + else + assert_nothing_raised { + server = FileBucket::BucketWebserver.new( + :Bucket => @bucket, + :Port => port + ) + } + assert_nothing_raised() { + trap(:INT) { server.shutdown } + serverthread = Thread.new { + server.start + } + } + end + + assert_nothing_raised { + client = FileBucket::Dipper.new( + :Server => "localhost", + :Port => port + ) + } + files.each { |file| + name = File.basename(file) + tmppath = File.join(tmpdir,name) + + # copy the files to our tmp directory so we can modify them... + #STDERR.puts("copying %s" % file) if $debug + File.open(tmppath,File::WRONLY|File::TRUNC|File::CREAT) { |wf| + File.open(file) { |rf| + wf.print(rf.read) + } + } + + assert(FileTest.exists?(tmppath)) + + osum = nil + tsum = nil + nsum = nil + assert_nothing_raised { + #STDERR.puts("backing up %s" % file) if $debug + osum = client.backup(file) + } + assert_nothing_raised { + #STDERR.puts("backing up %s" % tmppath) if $debug + tsum = client.backup(tmppath) + } + + assert(tsum == osum) + + File.open(tmppath,File::WRONLY|File::TRUNC) { |wf| + wf.print "This is some test text\n" + } + assert_nothing_raised { + #STDERR.puts("backing up %s" % tmppath) if $debug + nsum = client.backup(tmppath) + } + + assert(tsum != nsum) + + assert_nothing_raised { + #STDERR.puts("restoring %s" % tmppath) if $debug + nsum = client.restore(tmppath,tsum) + } + + assert_equal(tsum, nsum) + + contents = File.open(tmppath) { |rf| + #STDERR.puts("reading %s" % tmppath) if $debug + rf.read + } + csum = Digest::MD5.hexdigest(contents) + assert(tsum == csum) + } + + if $external + unless $pid + raise "Uh, we don't have a child pid" + end + system("kill %s" % $pid) + else + server.shutdown + + # make sure everything's complete before we stop + assert_nothing_raised() { + serverthread.join(60) + } + end + end +end diff --git a/test/types/tc_file.rb b/test/types/tc_file.rb index c646e6272..740f6394c 100644 --- a/test/types/tc_file.rb +++ b/test/types/tc_file.rb @@ -24,7 +24,7 @@ class TestFile < Test::Unit::TestCase return file end - def testfile + def mktestfile # because luke's home directory is on nfs, it can't be used for testing # as root tmpfile = tempfile() @@ -68,7 +68,7 @@ class TestFile < Test::Unit::TestCase end def test_zzowner - file = testfile() + file = mktestfile() users = {} count = 0 @@ -152,7 +152,7 @@ class TestFile < Test::Unit::TestCase end def test_group - file = testfile() + file = mktestfile() [%x{groups}.chomp.split(/ /), Process.groups].flatten.each { |group| assert_nothing_raised() { file[:group] = group @@ -224,7 +224,7 @@ class TestFile < Test::Unit::TestCase end def test_modes - file = testfile + file = mktestfile [0644,0755,0777,0641].each { |mode| assert_nothing_raised() { file[:mode] = mode diff --git a/test/types/tc_filebucket.rb b/test/types/tc_filebucket.rb new file mode 100755 index 000000000..e9644912d --- /dev/null +++ b/test/types/tc_filebucket.rb @@ -0,0 +1,117 @@ +if __FILE__ == $0 + $:.unshift '..' + $:.unshift '../../lib' + $puppetbase = "../../../../language/trunk" +end + +require 'puppet' +require 'test/unit' +require 'fileutils' +require 'puppettest' + +# $Id$ + +class TestFileBucket < Test::Unit::TestCase + include FileTesting + # hmmm + # this is complicated, because we store references to the created + # objects in a central store + def mkfile(hash) + file = nil + assert_nothing_raised { + file = Puppet::Type::PFile.new(hash) + } + return file + end + + def mktestfile + # because luke's home directory is on nfs, it can't be used for testing + # as root + tmpfile = tempfile() + File.open(tmpfile, "w") { |f| f.puts rand(100) } + @@tmpfiles.push tmpfile + mkfile(:name => tmpfile) + end + + def setup + @@tmpfiles = [] + Puppet[:loglevel] = :debug if __FILE__ == $0 + Puppet[:statefile] = "/var/tmp/puppetstate" + begin + initstorage + rescue + system("rm -rf %s" % Puppet[:statefile]) + end + end + + def teardown + clearstorage + Puppet::Type.allclear + @@tmpfiles.each { |file| + if FileTest.exists?(file) + system("chmod -R 755 %s" % file) + system("rm -rf %s" % file) + end + } + @@tmpfiles.clear + system("rm -f %s" % Puppet[:statefile]) + end + + def initstorage + Puppet::Storage.init + Puppet::Storage.load + end + + def clearstorage + Puppet::Storage.store + Puppet::Storage.clear + end + + def test_simplebucket + name = "yayness" + assert_nothing_raised { + Puppet::Type::PFileBucket.new( + :name => name, + :path => "/tmp/filebucket" + ) + } + + @@tmpfiles.push "/tmp/filebucket" + + bucket = nil + assert_nothing_raised { + bucket = Puppet::Type::PFileBucket.bucket(name) + } + + assert_instance_of(FileBucket::Dipper, bucket) + + Puppet.debug(bucket) + + md5 = nil + newpath = "/tmp/passwd" + system("cp /etc/passwd %s" % newpath) + assert_nothing_raised { + md5 = bucket.backup(newpath) + } + + assert(md5) + + newmd5 = nil + + File.open(newpath, "w") { |f| f.puts ";lkjasdf;lkjasdflkjwerlkj134lkj" } + + assert_nothing_raised { + newmd5 = bucket.backup(newpath) + } + + assert(md5 != newmd5) + + assert_nothing_raised { + bucket.restore(newpath, md5) + } + + File.open(newpath) { |f| newmd5 = Digest::MD5.hexdigest(f.read) } + + assert_equal(md5, newmd5) + end +end |