basic filebucketing is now working

git-svn-id: https://reductivelabs.com/svn/puppet/library/trunk@479 980ebf18-57e1-0310-9a29-db15c13687c0

6 files changed, 715 insertions, 11 deletions

diff --git a/lib/puppet/filebucket.rb b/lib/puppet/filebucket.rb
new file mode 100755
index 000000000..8a613359c
--- /dev/null
+++ b/lib/puppet/filebucket.rb
@@ -0,0 +1,273 @@
+#!/usr/bin/ruby -w
+
+#--------------------
+# accept and serve files
+#
+# $Id$
+
+
+require 'webrick'
+#require 'webrick/https'
+require 'xmlrpc/server'
+require 'xmlrpc/client'
+#require 'webrick/httpstatus'
+require 'facter'
+require 'digest/md5'
+require 'base64'
+
+class BucketError < RuntimeError; end
+
+module FileBucket
+    DEFAULTPORT = 8139
+    # this doesn't work for relative paths
+    def FileBucket.mkdir(dir)
+        if FileTest.exist?(dir)
+            return false
+        else
+            tmp = dir.sub(/^\//,'')
+            path = [File::SEPARATOR]
+            tmp.split(File::SEPARATOR).each { |dir|
+                path.push dir
+                unless FileTest.exist?(File.join(path))
+                    Dir.mkdir(File.join(path))
+                end
+            }
+            return true
+        end
+    end
+
+    def FileBucket.paths(base,md5)
+        return [
+            File.join(base, md5),
+            File.join(base, md5, "contents"),
+            File.join(base, md5, "paths")
+        ]
+    end
+
+    #---------------------------------------------------------------
+    class Bucket
+        def initialize(hash)
+            # build our AST
+
+            if hash.include?(:ConflictCheck)
+                @conflictchk = hash[:ConflictCheck]
+                hash.delete(:ConflictCheck)
+            else
+                @conflictchk = true
+            end
+
+            if hash.include?(:Path)
+                @bucket = hash[:Path]
+                hash.delete(:Path)
+            else
+                @bucket = "/var/puppet/filebucket"
+            end
+
+            # XXX this should really be done using Puppet::Type instances...
+            FileBucket.mkdir(@bucket)
+        end
+
+        # accept a file from a client
+        def addfile(string,path)
+            #puts "entering addfile"
+            contents = Base64.decode64(string)
+            #puts "string is decoded"
+
+            md5 = Digest::MD5.hexdigest(contents)
+            #puts "md5 is made"
+
+            bpath, bfile, pathpath = FileBucket.paths(@bucket,md5)
+
+            # if it's a new directory...
+            if FileBucket.mkdir(bpath)
+                # ...then just create the file
+                #puts "creating file"
+                File.open(bfile, File::WRONLY|File::CREAT) { |of|
+                    of.print contents
+                }
+                #puts "File is created"
+            else # if the dir already existed...
+                # ...we need to verify that the contents match the existing file
+                if @conflictchk
+                    unless FileTest.exists?(bfile)
+                        raise(BucketError,
+                            "No file at %s for sum %s" % [bfile,md5], caller)
+                    end
+
+                    curfile = ""
+                    File.open(bfile) { |of|
+                        curfile = of.read
+                    }
+
+                    # if the contents don't match, then we've found a conflict
+                    # unlikely, but quite bad
+                    if curfile != contents
+                        raise(BucketError,
+                            "Got passed new contents for sum %s" % md5, caller)
+                    end
+                end
+                #puts "Conflict check is done"
+            end
+
+            # in either case, add the passed path to the list of paths
+            paths = nil
+            addpath = false
+            if FileTest.exists?(pathpath)
+                File.open(pathpath) { |of|
+                    paths = of.readlines
+                }
+
+                # unless our path is already there...
+                unless paths.include?(path)
+                    addpath = true
+                end
+            else
+                addpath = true
+            end
+            #puts "Path is checked"
+
+            # if it's a new file, or if our path isn't in the file yet, add it
+            if addpath
+                File.open(pathpath, File::WRONLY|File::CREAT|File::APPEND) { |of|
+                    of.puts path
+                }
+                #puts "Path is added"
+            end
+
+            return md5
+        end
+
+        def getfile(md5)
+            bpath, bfile, bpaths = FileBucket.paths(@bucket,md5)
+
+            unless FileTest.exists?(bfile)
+                return false
+            end
+
+            contents = nil
+            File.open(bfile) { |of|
+                contents = of.read
+            }
+
+            return Base64.encode64(contents)
+        end
+
+        private
+
+        def on_init
+            @default_namespace = 'urn:filebucket-server'
+            add_method(self, 'addfile', 'string', 'path')
+            add_method(self, 'getfile', 'md5')
+        end
+
+        def cert(filename)
+            OpenSSL::X509::Certificate.new(File.open(File.join(@dir, filename)) { |f|
+                f.read
+            })
+        end
+
+        def key(filename)
+            OpenSSL::PKey::RSA.new(File.open(File.join(@dir, filename)) { |f|
+                f.read
+            })
+        end
+
+    end
+    #---------------------------------------------------------------
+
+    class BucketWebserver < WEBrick::HTTPServer
+        def initialize(hash)
+            unless hash.include?(:Port)
+                hash[:Port] = FileBucket::DEFAULTPORT
+            end
+            servlet = XMLRPC::WEBrickServlet.new
+            @bucket = FileBucket::Bucket.new(hash)
+            #puts @bucket
+            servlet.add_handler("bucket",@bucket)
+            super
+
+            self.mount("/RPC2", servlet)
+        end
+    end
+
+    class BucketClient < XMLRPC::Client
+        @@methods = [ :addfile, :getfile ]
+
+        @@methods.each { |method|
+            self.send(:define_method,method) { |*args|
+                begin
+                    call("bucket.%s" % method.to_s,*args)
+                rescue => detail
+                    #puts detail
+                end
+            }
+        }
+
+        def initialize(hash)
+            hash[:Path] ||= "/RPC2"
+            hash[:Server] ||= "localhost"
+            hash[:Port] ||= FileBucket::DEFAULTPORT
+            super(hash[:Server],hash[:Path],hash[:Port])
+        end
+    end
+
+    class Dipper
+        def initialize(hash)
+            if hash.include?(:Server)
+                @bucket = FileBucket::BucketClient.new(
+                    :Server => hash[:Server]
+                )
+            elsif hash.include?(:Bucket)
+                @bucket = hash[:Bucket]
+            elsif hash.include?(:Path)
+                @bucket = FileBucket::Bucket.new(
+                    :Path => hash[:Path]
+                )
+            end
+        end
+
+        def backup(file)
+            unless FileTest.exists?(file)
+                raise(BucketError, "File %s does not exist" % file, caller)
+            end
+            contents = File.open(file) { |of| of.read }
+
+            string = Base64.encode64(contents)
+            #puts "string is created"
+
+            sum = @bucket.addfile(string,file)
+            #puts "file %s is added" % file
+            return sum
+        end
+
+        def restore(file,sum)
+            restore = true
+            if FileTest.exists?(file)
+                contents = File.open(file) { |of| of.read }
+
+                cursum = Digest::MD5.hexdigest(contents)
+
+                # if the checksum has changed...
+                # this might be extra effort
+                if cursum == sum
+                    restore = false
+                end
+            end
+
+            if restore
+                #puts "Restoring %s" % file
+                newcontents = Base64.decode64(@bucket.getfile(sum))
+                newsum = Digest::MD5.hexdigest(newcontents)
+                File.open(file,File::WRONLY|File::TRUNC) { |of|
+                    of.print(newcontents)
+                }
+                #puts "Done"
+                return newsum
+            else
+                return nil
+            end
+
+        end
+    end
+    #---------------------------------------------------------------
+end
diff --git a/lib/puppet/type.rb b/lib/puppet/type.rb
index 2d64740c7..4f58a971c 100644
--- a/lib/puppet/type.rb
+++ b/lib/puppet/type.rb
@@ -1078,6 +1078,7 @@ end
 
 require 'puppet/type/service'
 require 'puppet/type/exec'
+require 'puppet/type/filebucket'
 require 'puppet/type/pfile'
 require 'puppet/type/symlink'
 require 'puppet/type/package'
diff --git a/lib/puppet/type/filebucket.rb b/lib/puppet/type/filebucket.rb
index e1b390e4d..537f3983b 100755
--- a/lib/puppet/type/filebucket.rb
+++ b/lib/puppet/type/filebucket.rb
@@ -5,29 +5,58 @@ require 'puppet/filebucket'
 
 module Puppet
 	class Type
-		class FileBucket < Type
+		class PFileBucket < Type
             attr_reader :bucket
 
 			@states = []
 
 			@parameters = [
                 :name,
-                :path
+                :server,
+                :path,
+                :port
             ]
 
             @name = :filebucket
 			@namevar = :name
 
+            @@buckets = {}
+
+            def self.bucket(name)
+                @@buckets[name]
+            end
+
             def initialize(hash)
                 super
 
-                unless self[:path] 
-                    self[:path] = File.join(Puppet[:puppetroot], "bucket")
+                if @parameters.include?(:server)
+                    @parameters[:port] ||= FileBucket::DEFAULTPORT
+                    begin
+                        @bucket = FileBucket::Dipper.new( 
+                            :Server => @parameters[:server],
+                            :Port => @parameters[:port]
+                        )
+                    rescue => detail
+                        raise Puppet::Error.new(
+                            "Could not create remote filebucket: %s" % detail
+                        )
+                    end
+                else
+                    @parameters[:path] ||= File.join(
+                        Puppet[:puppetroot], "bucket"
+                    )
+                    begin
+                        @bucket = FileBucket::Dipper.new(
+                            :Path => @parameters[:path]
+                        )
+                    rescue => detail
+                        raise Puppet::Error.new(
+                            "Could not create local filebucket: %s" % detail
+                        )
+                    end
                 end
 
-                @bucket = FileBucket::Bucket.new(
-                    :Bucket => self[:path]
-                )
+                @@buckets[self.name] = @bucket
             end
 
 		end # Puppet::Type::Service
diff --git a/test/bucket/tc_bucket.rb b/test/bucket/tc_bucket.rb
new file mode 100644
index 000000000..3aa045fc1
--- /dev/null
+++ b/test/bucket/tc_bucket.rb
@@ -0,0 +1,284 @@
+if __FILE__ == $0
+    $:.unshift '../../lib'
+    $:.unshift '../../../../library/trunk/lib/'
+    $:.unshift '../../../../library/trunk/test/'
+    $puppetbase = "../.."
+    $debug = true
+else
+    $debug = false
+end
+
+require 'puppet/filebucket'
+require 'test/unit'
+require 'puppettest.rb'
+require 'base64'
+
+# $Id$
+
+
+$external = true
+if ARGV[1] and ARGV[1] == "external"
+    $external = true
+else
+    # default to external
+    #$external = false
+end
+class TestBucket < Test::Unit::TestCase
+    def debug(string)
+        if $debug
+            puts([Time.now,string].join(" "))
+        end
+    end
+
+    def filelist
+        files = []
+            #/etc/passwd /etc/syslog.conf /etc/hosts
+        %w{
+            who /tmp/bigfile sh uname /etc/passwd /etc/syslog.conf /etc/hosts 
+        }.each { |file|
+            if file =~ /^\//
+                if FileTest.exists?(file)
+                    files.push file
+                end
+            else
+                begin
+                    path = %x{which #{file}}
+                rescue => detail
+                    #STDERR.puts "Could not search for binaries: %s" % detail
+                    next
+                end
+
+                if path != ""
+                    files.push path.chomp
+                end
+            end
+        }
+
+        return files
+    end
+
+    def setup
+        @bucket = File::SEPARATOR + File.join("tmp","filebuckettesting")
+    end
+
+    def teardown
+        system("rm -rf %s" % @bucket)
+        if defined? $pid
+            system("kill -9 #{$pid} 2>/dev/null")
+        end
+    end
+
+    def test_localserver
+        files = filelist()
+        server =nil
+        assert_nothing_raised {
+            server = FileBucket::Bucket.new(
+                :Bucket => @bucket
+            )
+        }
+        files.each { |file|
+            contents = File.open(file) { |of| of.read }
+
+            md5 = nil
+            assert_nothing_raised {
+                #STDERR.puts("adding %s" % file) if $debug
+                md5 = server.addfile(Base64.encode64(contents),file)
+            }
+            newcontents = nil
+            assert_nothing_raised {
+                #STDERR.puts("getting %s" % file) if $debug
+                newcontents = Base64.decode64(server.getfile(md5))
+            }
+
+            assert(
+                contents == newcontents
+            )
+        }
+    end
+
+    def test_localboth
+        files = filelist()
+
+        tmpdir = File.join(@bucket,"tmpfiledir")
+        FileBucket.mkdir(tmpdir)
+
+        server = nil
+        client = nil
+        threads = []
+        assert_nothing_raised {
+            server = FileBucket::Bucket.new(
+                :Bucket => @bucket
+            )
+        }
+
+        #sleep(30)
+        assert_nothing_raised {
+            client = FileBucket::Dipper.new(
+                :Bucket => server
+            )
+        }
+        files.each { |file|
+            name = File.basename(file)
+            tmppath = File.join(tmpdir,name)
+
+            # copy the files to our tmp directory so we can modify them...
+            #STDERR.puts("copying %s" % file) if $debug
+            File.open(tmppath,File::WRONLY|File::TRUNC|File::CREAT) { |wf|
+                File.open(file) { |rf|
+                    wf.print(rf.read)
+                }
+            }
+
+            assert(FileTest.exists?(tmppath))
+
+            osum = nil
+            tsum = nil
+            nsum = nil
+            assert_nothing_raised {
+                #STDERR.puts("backing up %s" % file) if $debug
+                osum = client.backup(file)
+            }
+            assert_nothing_raised {
+                #STDERR.puts("backing up %s" % tmppath) if $debug
+                tsum = client.backup(tmppath)
+            }
+
+            assert(tsum == osum)
+
+            File.open(tmppath,File::WRONLY|File::TRUNC) { |wf|
+                wf.print "This is some test text\n"
+            }
+            assert_nothing_raised {
+                #STDERR.puts("backing up %s" % tmppath) if $debug
+                nsum = client.backup(tmppath)
+            }
+
+            assert(tsum != nsum)
+
+            assert_nothing_raised {
+                #STDERR.puts("restoring %s" % tmppath) if $debug
+                nsum = client.restore(tmppath,tsum)
+            }
+
+            contents = File.open(tmppath) { |rf|
+                #STDERR.puts("reading %s" % tmppath) if $debug
+                rf.read
+            }
+            csum = Digest::MD5.hexdigest(contents)
+            assert(tsum == csum)
+        }
+    end
+
+    def test_webxmlmix
+        files = filelist()
+
+        tmpdir = File.join(@bucket,"tmpfiledir")
+        FileBucket.mkdir(tmpdir)
+
+        server = nil
+        client = nil
+        port = FileBucket::DEFAULTPORT
+        serverthread = nil
+        pid = nil
+        if $external
+            $pid = fork {
+                server = FileBucket::BucketWebserver.new(
+                    :Bucket => @bucket,
+                    :Port => port
+                )
+                trap(:INT) { server.shutdown }
+                trap(:TERM) { server.shutdown }
+                server.start
+            }
+            sleep 3
+            #puts "pid is %s" % pid
+            #exit
+        else
+            assert_nothing_raised {
+                server = FileBucket::BucketWebserver.new(
+                    :Bucket => @bucket,
+                    :Port => port
+                )
+            }
+            assert_nothing_raised() {
+                trap(:INT) { server.shutdown }
+                serverthread = Thread.new {
+                    server.start
+                }
+            }
+        end
+
+        assert_nothing_raised {
+            client = FileBucket::Dipper.new(
+                :Server => "localhost",
+                :Port => port
+            )
+        }
+        files.each { |file|
+            name = File.basename(file)
+            tmppath = File.join(tmpdir,name)
+
+            # copy the files to our tmp directory so we can modify them...
+            #STDERR.puts("copying %s" % file) if $debug
+            File.open(tmppath,File::WRONLY|File::TRUNC|File::CREAT) { |wf|
+                File.open(file) { |rf|
+                    wf.print(rf.read)
+                }
+            }
+
+            assert(FileTest.exists?(tmppath))
+
+            osum = nil
+            tsum = nil
+            nsum = nil
+            assert_nothing_raised {
+                #STDERR.puts("backing up %s" % file) if $debug
+                osum = client.backup(file)
+            }
+            assert_nothing_raised {
+                #STDERR.puts("backing up %s" % tmppath) if $debug
+                tsum = client.backup(tmppath)
+            }
+
+            assert(tsum == osum)
+
+            File.open(tmppath,File::WRONLY|File::TRUNC) { |wf|
+                wf.print "This is some test text\n"
+            }
+            assert_nothing_raised {
+                #STDERR.puts("backing up %s" % tmppath) if $debug
+                nsum = client.backup(tmppath)
+            }
+
+            assert(tsum != nsum)
+
+            assert_nothing_raised {
+                #STDERR.puts("restoring %s" % tmppath) if $debug
+                nsum = client.restore(tmppath,tsum)
+            }
+
+            assert_equal(tsum, nsum)
+
+            contents = File.open(tmppath) { |rf|
+                #STDERR.puts("reading %s" % tmppath) if $debug
+                rf.read
+            }
+            csum = Digest::MD5.hexdigest(contents)
+            assert(tsum == csum)
+        }
+
+        if $external
+            unless $pid
+                raise "Uh, we don't have a child pid"
+            end
+            system("kill %s" % $pid)
+        else
+            server.shutdown
+
+            # make sure everything's complete before we stop
+            assert_nothing_raised() {
+                serverthread.join(60)
+            }
+        end
+    end
+end
diff --git a/test/types/tc_file.rb b/test/types/tc_file.rb
index c646e6272..740f6394c 100644
--- a/test/types/tc_file.rb
+++ b/test/types/tc_file.rb
@@ -24,7 +24,7 @@ class TestFile < Test::Unit::TestCase
         return file
     end
 
-    def testfile
+    def mktestfile
         # because luke's home directory is on nfs, it can't be used for testing
         # as root
         tmpfile = tempfile()
@@ -68,7 +68,7 @@ class TestFile < Test::Unit::TestCase
     end
 
     def test_zzowner
-        file = testfile()
+        file = mktestfile()
 
         users = {}
         count = 0
@@ -152,7 +152,7 @@ class TestFile < Test::Unit::TestCase
     end
 
     def test_group
-        file = testfile()
+        file = mktestfile()
         [%x{groups}.chomp.split(/ /), Process.groups].flatten.each { |group|
             assert_nothing_raised() {
                 file[:group] = group
@@ -224,7 +224,7 @@ class TestFile < Test::Unit::TestCase
     end
 
     def test_modes
-        file = testfile
+        file = mktestfile
         [0644,0755,0777,0641].each { |mode|
             assert_nothing_raised() {
                 file[:mode] = mode
diff --git a/test/types/tc_filebucket.rb b/test/types/tc_filebucket.rb
new file mode 100755
index 000000000..e9644912d
--- /dev/null
+++ b/test/types/tc_filebucket.rb
@@ -0,0 +1,117 @@
+if __FILE__ == $0
+    $:.unshift '..'
+    $:.unshift '../../lib'
+    $puppetbase = "../../../../language/trunk"
+end
+
+require 'puppet'
+require 'test/unit'
+require 'fileutils'
+require 'puppettest'
+
+# $Id$
+
+class TestFileBucket < Test::Unit::TestCase
+    include FileTesting
+    # hmmm
+    # this is complicated, because we store references to the created
+    # objects in a central store
+    def mkfile(hash)
+        file = nil
+        assert_nothing_raised {
+            file = Puppet::Type::PFile.new(hash)
+        }
+        return file
+    end
+
+    def mktestfile
+        # because luke's home directory is on nfs, it can't be used for testing
+        # as root
+        tmpfile = tempfile()
+        File.open(tmpfile, "w") { |f| f.puts rand(100) }
+        @@tmpfiles.push tmpfile
+        mkfile(:name => tmpfile)
+    end
+
+    def setup
+        @@tmpfiles = []
+        Puppet[:loglevel] = :debug if __FILE__ == $0
+        Puppet[:statefile] = "/var/tmp/puppetstate"
+        begin
+            initstorage
+        rescue
+            system("rm -rf %s" % Puppet[:statefile])
+        end
+    end
+
+    def teardown
+        clearstorage
+        Puppet::Type.allclear
+        @@tmpfiles.each { |file|
+            if FileTest.exists?(file)
+                system("chmod -R 755 %s" % file)
+                system("rm -rf %s" % file)
+            end
+        }
+        @@tmpfiles.clear
+        system("rm -f %s" % Puppet[:statefile])
+    end
+
+    def initstorage
+        Puppet::Storage.init
+        Puppet::Storage.load
+    end
+
+    def clearstorage
+        Puppet::Storage.store
+        Puppet::Storage.clear
+    end
+
+    def test_simplebucket
+        name = "yayness"
+        assert_nothing_raised {
+            Puppet::Type::PFileBucket.new(
+                :name => name,
+                :path => "/tmp/filebucket"
+            )
+        }
+
+        @@tmpfiles.push "/tmp/filebucket"
+
+        bucket = nil
+        assert_nothing_raised {
+            bucket = Puppet::Type::PFileBucket.bucket(name)
+        }
+
+        assert_instance_of(FileBucket::Dipper, bucket)
+
+        Puppet.debug(bucket)
+
+        md5 = nil
+        newpath = "/tmp/passwd"
+        system("cp /etc/passwd %s" % newpath)
+        assert_nothing_raised {
+            md5 = bucket.backup(newpath)
+        }
+
+        assert(md5)
+
+        newmd5 = nil
+
+        File.open(newpath, "w") { |f| f.puts ";lkjasdf;lkjasdflkjwerlkj134lkj" }
+
+        assert_nothing_raised {
+            newmd5 = bucket.backup(newpath)
+        }
+
+        assert(md5 != newmd5)
+
+        assert_nothing_raised {
+            bucket.restore(newpath, md5)
+        }
+
+        File.open(newpath) { |f| newmd5 = Digest::MD5.hexdigest(f.read) }
+
+        assert_equal(md5, newmd5)
+    end
+end