Merge branch '2.6.x' into next

* 2.6.x: (21 commits)
  (#5900) Include ResourceStatus#failed in serialized reports
  (#5882) Added error-handling for bucketing files in puppet inspect
  (#5882) Added error-handling to puppet inspect when auditing
  (#5171) Made "puppet inspect" upload audited files to a file bucket
  Prep for #5171: Added a missing require to inspect application.
  Locked Puppet license to GPLv2
  (#5838) Support paths as part of file bucket requests.
  (#5838) Improve the quality of file bucket specs.
  (#5838) Make file bucket dipper efficient when saving a file that already exists
  (#5838) Implemented the "head" method for FileBucketFile::File terminus.
  (#5838) Reworked file dipper spec to perform less stubbing.
  (#5838) Added support for HEAD requests to the indirector.
  (#5838) Refactored error handling logic into find_in_cache.
  (#5838) Refactored Puppet::Network::Rights#fail_on_deny
  maint: Remove unused Rakefile in spec directory
  (#5171) Made filebucket able to perform diffs
  (#5710) Removed unnecessary calls to insync?
  Prep for fixing #5710: Refactor stub provider in resource harness spec
  Maint: test partial resource failure
  maint: Inspect reports should have audited = true on events
  ...

Manually Resolved Conflicts:
	lib/puppet/file_bucket/dipper.rb
	lib/puppet/indirector.rb
	lib/puppet/network/rest_authconfig.rb
	spec/unit/file_bucket/dipper_spec.rb
	spec/unit/file_bucket/file_spec.rb
	spec/unit/indirector_spec.rb

3 files changed, 77 insertions, 90 deletions

diff --git a/lib/puppet/indirector/file_bucket_file/file.rb b/lib/puppet/indirector/file_bucket_file/file.rb
index 318858aaf..8bea2d767 100644
--- a/lib/puppet/indirector/file_bucket_file/file.rb
+++ b/lib/puppet/indirector/file_bucket_file/file.rb
@@ -14,16 +14,31 @@ module Puppet::FileBucketFile
     end
 
     def find( request )
-      checksum, path = request_to_checksum_and_path( request )
-      find_by_checksum( checksum, request.options )
+      checksum = request_to_checksum( request )
+      file_path = path_for(request.options[:bucket_path], checksum, 'contents')
+
+      return nil unless ::File.exists?(file_path)
+
+      if request.options[:diff_with]
+        hash_protocol = sumtype(checksum)
+        file2_path = path_for(request.options[:bucket_path], request.options[:diff_with], 'contents')
+        raise "could not find diff_with #{request.options[:diff_with]}" unless ::File.exists?(file2_path)
+        return `diff #{file_path.inspect} #{file2_path.inspect}`
+      else
+        contents = ::File.read file_path
+        Puppet.info "FileBucket read #{checksum}"
+        model.new(contents)
+      end
     end
 
-    def save( request )
-      checksum, path = request_to_checksum_and_path( request )
+    def head(request)
+      checksum = request_to_checksum(request)
+      file_path = path_for(request.options[:bucket_path], checksum, 'contents')
+      ::File.exists?(file_path)
+    end
 
+    def save( request )
       instance = request.instance
-      instance.checksum = checksum if checksum
-      instance.path = path if path
 
       save_to_disk(instance)
       instance.to_s
@@ -31,66 +46,41 @@ module Puppet::FileBucketFile
 
     private
 
-    def find_by_checksum( checksum, options )
-      model.new( nil, :checksum => checksum ) do |bucket_file|
-        bucket_file.bucket_path = options[:bucket_path]
-        filename = contents_path_for( bucket_file )
-
-        return nil if ! ::File.exist? filename
-
-        begin
-          contents = ::File.read filename
-          Puppet.info "FileBucket read #{bucket_file.checksum}"
-        rescue RuntimeError => e
-          raise Puppet::Error, "file could not be read: #{e.message}"
-        end
-
-        if ::File.exist?(paths_path_for( bucket_file) )
-          ::File.open(paths_path_for( bucket_file) ) do |f|
-            bucket_file.paths = f.readlines.map { |l| l.chomp }
-          end
-        end
-
-        bucket_file.contents = contents
-      end
-    end
-
     def save_to_disk( bucket_file )
-      # If the file already exists, just return the md5 sum.
-      if ::File.exist?(contents_path_for( bucket_file) )
+      filename = path_for(bucket_file.bucket_path, bucket_file.checksum_data, 'contents')
+      dirname = path_for(bucket_file.bucket_path, bucket_file.checksum_data)
+
+      # If the file already exists, do nothing.
+      if ::File.exist?(filename)
         verify_identical_file!(bucket_file)
       else
         # Make the directories if necessary.
-        unless ::File.directory?( path_for( bucket_file) )
+        unless ::File.directory?(dirname)
           Puppet::Util.withumask(0007) do
-            ::FileUtils.mkdir_p( path_for( bucket_file) )
+            ::FileUtils.mkdir_p(dirname)
           end
         end
 
-        Puppet.info "FileBucket adding #{bucket_file.path} as #{bucket_file.checksum}"
+        Puppet.info "FileBucket adding #{bucket_file.checksum}"
 
         # Write the file to disk.
         Puppet::Util.withumask(0007) do
-          ::File.open(contents_path_for(bucket_file), ::File::WRONLY|::File::CREAT, 0440) do |of|
+          ::File.open(filename, ::File::WRONLY|::File::CREAT, 0440) do |of|
             of.print bucket_file.contents
           end
         end
       end
-
-      save_path_to_paths_file(bucket_file)
-      bucket_file.checksum_data
     end
 
-    def request_to_checksum_and_path( request )
-      return [request.key, nil] if checksum?(request.key)
-
-      checksum_type, checksum, path = request.key.split(/\//, 3)
-      return(checksum_type.to_s == "" ? nil : [ "{#{checksum_type}}#{checksum}", path ])
+    def request_to_checksum( request )
+      checksum_type, checksum, path = request.key.split(/\//, 3) # Note: we ignore path if present.
+      raise "Unsupported checksum type #{checksum_type.inspect}" if checksum_type != 'md5'
+      raise "Invalid checksum #{checksum.inspect}" if checksum !~ /^[0-9a-f]{32}$/
+      checksum
     end
 
-    def path_for(bucket_file, subfile = nil)
-      bucket_path = bucket_file.bucket_path || Puppet[:bucketdir]
-      digest      = bucket_file.checksum_data
+    def path_for(bucket_path, digest, subfile = nil)
+      bucket_path ||= Puppet[:bucketdir]
 
       dir     = ::File.join(digest[0..7].split(""))
       basedir = ::File.join(bucket_path, dir, digest)
@@ -99,48 +89,18 @@ module Puppet::FileBucketFile
       ::File.join(basedir, subfile)
     end
 
-    def contents_path_for(bucket_file)
-      path_for(bucket_file, "contents")
-    end
-
-    def paths_path_for(bucket_file)
-      path_for(bucket_file, "paths")
-    end
-
-    def content_check?
-      true
-    end
-
     # If conflict_check is enabled, verify that the passed text is
     # the same as the text in our file.
     def verify_identical_file!(bucket_file)
-      return unless content_check?
-      disk_contents = ::File.read(contents_path_for(bucket_file))
+      disk_contents = ::File.read(path_for(bucket_file.bucket_path, bucket_file.checksum_data, 'contents'))
 
       # If the contents don't match, then we've found a conflict.
       # Unlikely, but quite bad.
       if disk_contents != bucket_file.contents
-        raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}", caller
+        raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}"
       else
-        Puppet.info "FileBucket got a duplicate file #{bucket_file.path} (#{bucket_file.checksum})"
+        Puppet.info "FileBucket got a duplicate file #{bucket_file.checksum}"
       end
     end
-
-    def save_path_to_paths_file(bucket_file)
-      return unless bucket_file.path
-
-      # check for dupes
-      if ::File.exist?(paths_path_for( bucket_file) )
-        ::File.open(paths_path_for( bucket_file) ) do |f|
-          return if f.readlines.collect { |l| l.chomp }.include?(bucket_file.path)
-        end
-      end
-
-      # if it's a new file, or if our path isn't in the file yet, add it
-      ::File.open(paths_path_for(bucket_file), ::File::WRONLY|::File::CREAT|::File::APPEND) do |of|
-        of.puts bucket_file.path
-      end
-    end
-
   end
 end
diff --git a/lib/puppet/indirector/indirection.rb b/lib/puppet/indirector/indirection.rb
index eb0aa8aee..d958a82ac 100644
--- a/lib/puppet/indirector/indirection.rb
+++ b/lib/puppet/indirector/indirection.rb
@@ -180,18 +180,13 @@ class Puppet::Indirector::Indirection
     request = request(:find, key, *args)
     terminus = prepare(request)
 
-    begin
-      if result = find_in_cache(request)
-        return result
-      end
-    rescue => detail
-      puts detail.backtrace if Puppet[:trace]
-      Puppet.err "Cached #{self.name} for #{request.key} failed: #{detail}"
+    if result = find_in_cache(request)
+      return result
     end
 
     # Otherwise, return the result from the terminus, caching if appropriate.
     if ! request.ignore_terminus? and result = terminus.find(request)
-      result.expiration ||= self.expiration
+      result.expiration ||= self.expiration if result.respond_to?(:expiration)
       if cache? and request.use_cache?
         Puppet.info "Caching #{self.name} for #{request.key}"
         cache.save request(:save, result, *args)
@@ -203,6 +198,17 @@ class Puppet::Indirector::Indirection
     nil
   end
 
+  # Search for an instance in the appropriate terminus, and return a
+  # boolean indicating whether the instance was found.
+  def head(key, *args)
+    request = request(:head, key, *args)
+    terminus = prepare(request)
+
+    # Look in the cache first, then in the terminus.  Force the result
+    # to be a boolean.
+    !!(find_in_cache(request) || terminus.head(request))
+  end
+
   def find_in_cache(request)
     # See if our instance is in the cache and up to date.
     return nil unless cache? and ! request.ignore_cache? and cached = cache.find(request)
@@ -213,6 +219,10 @@ class Puppet::Indirector::Indirection
 
     Puppet.debug "Using cached #{self.name} for #{request.key}"
     cached
+  rescue => detail
+    puts detail.backtrace if Puppet[:trace]
+    Puppet.err "Cached #{self.name} for #{request.key} failed: #{detail}"
+    nil
   end
 
   # Remove something via the terminus.
diff --git a/lib/puppet/indirector/rest.rb b/lib/puppet/indirector/rest.rb
index eb41ff3b1..e50dc68ae 100644
--- a/lib/puppet/indirector/rest.rb
+++ b/lib/puppet/indirector/rest.rb
@@ -53,11 +53,15 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus
       end
     else
       # Raise the http error if we didn't get a 'success' of some kind.
-      message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
-      raise Net::HTTPError.new(message, response)
+      raise convert_to_http_error(response)
     end
   end
 
+  def convert_to_http_error(response)
+    message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
+    Net::HTTPError.new(message, response)
+  end
+
   # Provide appropriate headers.
   def headers
     add_accept_encoding({"Accept" => model.supported_formats.join(", ")})
@@ -73,6 +77,19 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus
     result
   end
 
+  def head(request)
+    response = network(request).head(indirection2uri(request), headers)
+    case response.code
+    when "404"
+      return false
+    when /^2/
+      return true
+    else
+      # Raise the http error if we didn't get a 'success' of some kind.
+      raise convert_to_http_error(response)
+    end
+  end
+
   def search(request)
     unless result = deserialize(network(request).get(indirection2uri(request), headers), true)
       return []