From dcce45cfcbf77d576237ae8ff0ffa6ef98dcb722 Mon Sep 17 00:00:00 2001 From: Paul Berry Date: Mon, 21 Feb 2011 12:50:53 -0800 Subject: (#6353) Restore the ability to store paths in the filebucket Commit 2274d5104f6e413a2b8899a3c3111a17bbb2f4d7 optimized network usage for the case where a file is already in the filebucket. However, it took away the ability to store paths. This change restores the ability to store paths while maintaining optimal network usage for the case where the file is already in the filebucket with the given path. This is expected to be the most common case. Paired-with: Jesse Wolfe --- lib/puppet/file_bucket/dipper.rb | 5 ++- lib/puppet/indirector/file_bucket_file/file.rb | 55 ++++++++++++++++++++------ 2 files changed, 45 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/puppet/file_bucket/dipper.rb b/lib/puppet/file_bucket/dipper.rb index f4bef28a8..de4c01b78 100644 --- a/lib/puppet/file_bucket/dipper.rb +++ b/lib/puppet/file_bucket/dipper.rb @@ -34,11 +34,12 @@ class Puppet::FileBucket::Dipper contents = ::File.read(file) begin file_bucket_file = Puppet::FileBucket::File.new(contents, :bucket_path => @local_path) - dest_path = "#{@rest_path}#{file_bucket_file.name}" + files_original_path = absolutize_path(file) + dest_path = "#{@rest_path}#{file_bucket_file.name}#{files_original_path}" # Make a HEAD request for the file so that we don't waste time # uploading it if it already exists in the bucket. - unless Puppet::FileBucket::File.head("#{@rest_path}#{file_bucket_file.checksum_type}/#{file_bucket_file.checksum_data}") + unless Puppet::FileBucket::File.head("#{@rest_path}#{file_bucket_file.checksum_type}/#{file_bucket_file.checksum_data}#{files_original_path}") file_bucket_file.save(dest_path) end diff --git a/lib/puppet/indirector/file_bucket_file/file.rb b/lib/puppet/indirector/file_bucket_file/file.rb index 8bea2d767..0fd8a914f 100644 --- a/lib/puppet/indirector/file_bucket_file/file.rb +++ b/lib/puppet/indirector/file_bucket_file/file.rb @@ -14,10 +14,12 @@ module Puppet::FileBucketFile end def find( request ) - checksum = request_to_checksum( request ) - file_path = path_for(request.options[:bucket_path], checksum, 'contents') + checksum, files_original_path = request_to_checksum_and_path( request ) + dir_path = path_for(request.options[:bucket_path], checksum) + file_path = ::File.join(dir_path, 'contents') return nil unless ::File.exists?(file_path) + return nil unless path_match(dir_path, files_original_path) if request.options[:diff_with] hash_protocol = sumtype(checksum) @@ -32,32 +34,47 @@ module Puppet::FileBucketFile end def head(request) - checksum = request_to_checksum(request) - file_path = path_for(request.options[:bucket_path], checksum, 'contents') - ::File.exists?(file_path) + checksum, files_original_path = request_to_checksum_and_path(request) + dir_path = path_for(request.options[:bucket_path], checksum) + + ::File.exists?(::File.join(dir_path, 'contents')) and path_match(dir_path, files_original_path) end def save( request ) instance = request.instance + checksum, files_original_path = request_to_checksum_and_path(request) - save_to_disk(instance) + save_to_disk(instance, files_original_path) instance.to_s end private - def save_to_disk( bucket_file ) + def path_match(dir_path, files_original_path) + return true unless files_original_path # if no path was provided, it's a match + paths_path = ::File.join(dir_path, 'paths') + return false unless ::File.exists?(paths_path) + ::File.open(paths_path) do |f| + f.each do |line| + return true if line.chomp == files_original_path + end + end + return false + end + + def save_to_disk( bucket_file, files_original_path ) filename = path_for(bucket_file.bucket_path, bucket_file.checksum_data, 'contents') - dirname = path_for(bucket_file.bucket_path, bucket_file.checksum_data) + dir_path = path_for(bucket_file.bucket_path, bucket_file.checksum_data) + paths_path = ::File.join(dir_path, 'paths') # If the file already exists, do nothing. if ::File.exist?(filename) verify_identical_file!(bucket_file) else # Make the directories if necessary. - unless ::File.directory?(dirname) + unless ::File.directory?(dir_path) Puppet::Util.withumask(0007) do - ::FileUtils.mkdir_p(dirname) + ::FileUtils.mkdir_p(dir_path) end end @@ -68,15 +85,27 @@ module Puppet::FileBucketFile ::File.open(filename, ::File::WRONLY|::File::CREAT, 0440) do |of| of.print bucket_file.contents end + ::File.open(paths_path, ::File::WRONLY|::File::CREAT, 0640) do |of| + # path will be written below + end + end + end + + unless path_match(dir_path, files_original_path) + ::File.open(paths_path, 'a') do |f| + f.puts(files_original_path) end end end - def request_to_checksum( request ) - checksum_type, checksum, path = request.key.split(/\//, 3) # Note: we ignore path if present. + def request_to_checksum_and_path( request ) + checksum_type, checksum, path = request.key.split(/\//, 3) + if path == '' # Treat "md5//" like "md5/" + path = nil + end raise "Unsupported checksum type #{checksum_type.inspect}" if checksum_type != 'md5' raise "Invalid checksum #{checksum.inspect}" if checksum !~ /^[0-9a-f]{32}$/ - checksum + [checksum, path] end def path_for(bucket_path, digest, subfile = nil) -- cgit