Feature #3347 REST-ified FileBucket

FileBucket Files have been reimplemented as an indirector terminus so that
they can be transmitted over REST.
The old Network::Client.dipper has been replaced with a compatibility later
in FileBucket::Dipper that uses the indirector to access filebucket termini.

Slightly revised patch:
* No longer allows nil contents in FileBucket outside of initialization
* Uses File.exist? instead of the deprecated File.exists?
* Tweaks JSON serialization and de-serialization to include "path"

Deferred issues:
* Feature #3371 "FileBucket should not keep files in memory".
* Feature #3372 "Replace FileBucket Dipper with more idiomatic calls"

3 files changed, 230 insertions, 0 deletions

diff --git a/lib/puppet/file_bucket/dipper.rb b/lib/puppet/file_bucket/dipper.rb
new file mode 100644
index 000000000..c73d76345
--- /dev/null
+++ b/lib/puppet/file_bucket/dipper.rb
@@ -0,0 +1,97 @@
+require 'puppet/file_bucket'
+require 'puppet/file_bucket/file'
+require 'puppet/indirector/request'
+
+class Puppet::FileBucket::Dipper
+    # This is a transitional implementation that uses REST
+    # to access remote filebucket files.
+
+    attr_accessor :name
+
+    # Create our bucket client
+    def initialize(hash = {})
+        # Emulate the XMLRPC client
+        server      = hash[:Server]
+        port        = hash[:Port] || Puppet[:masterport]
+        environment = Puppet[:environment]
+
+        if hash.include?(:Path)
+            @local_path = hash[:Path]
+            @rest_path  = nil
+        else
+            @local_path = nil
+            @rest_path = "https://#{server}:#{port}/#{environment}/file_bucket_file/"
+        end
+    end
+
+    def local?
+        !! @local_path
+    end
+
+    # Back up a file to our bucket
+    def backup(file)
+        unless ::File.exist?(file)
+            raise(ArgumentError, "File %s does not exist" % file)
+        end
+        contents = ::File.read(file)
+        begin
+            file_bucket_file = Puppet::FileBucket::File.new(contents, :bucket_path => @local_path, :path => file)
+            dest_path = "#{@rest_path}#{file_bucket_file.name}"
+
+            request = Puppet::Indirector::Request.new(:file_bucket_file, :save, dest_path)
+
+            file_bucket_file.save(request)
+            return file_bucket_file.checksum_data
+        rescue => detail
+            puts detail.backtrace if Puppet[:trace]
+            raise Puppet::Error, "Could not back up %s: %s" % [file, detail]
+        end
+    end
+
+    # Retrieve a file by sum.
+    def getfile(sum)
+        source_path = "#{@rest_path}md5/#{sum}"
+        file_bucket_file = Puppet::FileBucket::File.find(source_path)
+
+        return file_bucket_file.to_s
+    end
+
+    # Restore the file
+    def restore(file,sum)
+        restore = true
+        if FileTest.exists?(file)
+            cursum = Digest::MD5.hexdigest(::File.read(file))
+
+            # if the checksum has changed...
+            # this might be extra effort
+            if cursum == sum
+                restore = false
+            end
+        end
+
+        if restore
+            if newcontents = getfile(sum)
+                tmp = ""
+                newsum = Digest::MD5.hexdigest(newcontents)
+                changed = nil
+                if FileTest.exists?(file) and ! FileTest.writable?(file)
+                    changed = ::File.stat(file).mode
+                    ::File.chmod(changed | 0200, file)
+                end
+                ::File.open(file, ::File::WRONLY|::File::TRUNC|::File::CREAT) { |of|
+                    of.print(newcontents)
+                }
+                if changed
+                    ::File.chmod(changed, file)
+                end
+            else
+                Puppet.err "Could not find file with checksum %s" % sum
+                return nil
+            end
+            return newsum
+        else
+            return nil
+        end
+    end
+end
+
diff --git a/lib/puppet/file_bucket/file.rb b/lib/puppet/file_bucket/file.rb
new file mode 100644
index 000000000..7122bfbbb
--- /dev/null
+++ b/lib/puppet/file_bucket/file.rb
@@ -0,0 +1,123 @@
+require 'puppet/file_bucket'
+require 'puppet/indirector'
+
+class Puppet::FileBucket::File
+    # This class handles the abstract notion of a file in a filebucket.
+    # There are mechanisms to save and load this file locally and remotely in puppet/indirector/filebucketfile/*
+    # There is a compatibility class that emulates pre-indirector filebuckets in Puppet::FileBucket::Dipper
+    extend Puppet::Indirector
+    require 'puppet/file_bucket/file/indirection_hooks'
+    indirects :file_bucket_file, :terminus_class => :file, :extend => Puppet::FileBucket::File::IndirectionHooks
+
+    attr :path, true
+    attr :paths, true
+    attr :contents, true
+    attr :checksum_type
+    attr :bucket_path
+
+    def self.default_checksum_type
+        :md5
+    end
+
+    def initialize( contents, options = {} )
+        @contents      = contents
+        @bucket_path   = options[:bucket_path]
+        @path          = options[:path]
+        @paths         = options[:paths] || []
+        @checksum      = options[:checksum]
+        @checksum_type = options[:checksum_type] || self.class.default_checksum_type
+
+        yield(self) if block_given?
+
+        validate!
+    end
+
+    def validate!
+        digest_class( @checksum_type ) # raises error on bad types
+        raise ArgumentError, 'contents must be a string' unless @contents.is_a?(String)
+        validate_checksum(@checksum) if @checksum
+    end
+
+    def contents=(contents)
+        raise "You may not change the contents of a FileBucket File" if @contents
+        @contents = contents
+    end
+
+    def checksum=(checksum)
+        validate_checksum(checksum)
+        self.checksum_type = checksum # this grabs the prefix only
+        @checksum = checksum
+    end
+
+    def validate_checksum(new_checksum)
+        unless new_checksum == checksum_of_type(new_checksum)
+            raise Puppet::Error, "checksum does not match contents"
+        end
+    end
+
+    def checksum
+        @checksum ||= checksum_of_type(checksum_type)
+    end
+
+    def checksum_of_type( type )
+        type = checksum_type( type ) # strip out data segment if there is one
+        type.to_s + ":" + digest_class(type).hexdigest(@contents)
+    end
+
+    def checksum_type=( new_checksum_type )
+        @checksum = nil
+        @checksum_type = checksum_type(new_checksum_type)
+    end
+
+    def checksum_type(checksum = @checksum_type)
+        checksum.to_s.split(':',2)[0].to_sym
+    end
+
+    def checksum_data(new_checksum = self.checksum)
+        new_checksum.split(':',2)[1]
+    end
+
+    def checksum_data=(new_data)
+        self.checksum = "#{checksum_type}:#{new_data}"
+    end
+
+    def digest_class(type = nil)
+        case checksum_type(type)
+        when :md5  : require 'digest/md5'  ; Digest::MD5
+        when :sha1 : require 'digest/sha1' ; Digest::SHA1
+        else
+            raise ArgumentError, "not a known checksum type: #{checksum_type(type)}"
+        end
+    end
+
+    def to_s
+        contents
+    end
+
+    def name
+        [checksum_type, checksum_data, path].compact.join('/')
+    end
+
+    def name=(name)
+        self.checksum_type, self.checksum_data, self.path = name.split('/',3)
+    end
+
+    def conflict_check?
+        true
+    end
+
+    def self.from_s( contents )
+        self.new( contents )
+    end
+
+    def to_pson
+        hash = { "contents" => contents }
+        hash["path"] = @path if @path
+        hash.to_pson
+    end
+
+    def self.from_pson( pson )
+        self.new( pson["contents"], :path => pson["path"] )
+    end
+
+end
diff --git a/lib/puppet/file_bucket/file/indirection_hooks.rb b/lib/puppet/file_bucket/file/indirection_hooks.rb
new file mode 100644
index 000000000..ec2bb3469
--- /dev/null
+++ b/lib/puppet/file_bucket/file/indirection_hooks.rb
@@ -0,0 +1,10 @@
+require 'puppet/file_bucket/file'
+
+# This module is used to pick the appropriate terminus
+# in filebucket indirections.
+module Puppet::FileBucket::File::IndirectionHooks
+    def select_terminus(request)
+        return :rest if request.protocol == 'https'
+        return Puppet::FileBucket::File.indirection.terminus_class
+    end
+end