diff options
Diffstat (limited to 'lib/puppet')
-rw-r--r-- | lib/puppet/application/filebucket.rb | 9 | ||||
-rw-r--r-- | lib/puppet/application/server.rb | 1 | ||||
-rw-r--r-- | lib/puppet/checksum.rb | 57 | ||||
-rw-r--r-- | lib/puppet/file_bucket.rb | 4 | ||||
-rw-r--r-- | lib/puppet/file_bucket/dipper.rb (renamed from lib/puppet/network/client/dipper.rb) | 52 | ||||
-rw-r--r-- | lib/puppet/file_bucket/file.rb | 123 | ||||
-rw-r--r-- | lib/puppet/file_bucket/file/indirection_hooks.rb | 10 | ||||
-rw-r--r-- | lib/puppet/indirector/file_bucket_file/file.rb | 142 | ||||
-rw-r--r-- | lib/puppet/indirector/file_bucket_file/rest.rb | 8 | ||||
-rw-r--r-- | lib/puppet/indirector/indirection.rb | 14 | ||||
-rw-r--r-- | lib/puppet/indirector/request.rb | 10 | ||||
-rwxr-xr-x | lib/puppet/network/handler/filebucket.rb | 139 | ||||
-rw-r--r-- | lib/puppet/network/http/handler.rb | 2 | ||||
-rwxr-xr-x | lib/puppet/type/filebucket.rb | 4 | ||||
-rwxr-xr-x | lib/puppet/type/tidy.rb | 2 |
15 files changed, 352 insertions, 225 deletions
diff --git a/lib/puppet/application/filebucket.rb b/lib/puppet/application/filebucket.rb index 0723054df..09aaf2b5d 100644 --- a/lib/puppet/application/filebucket.rb +++ b/lib/puppet/application/filebucket.rb @@ -1,6 +1,6 @@ require 'puppet' require 'puppet/application' -require 'puppet/network/client' +require 'puppet/file_bucket/dipper' Puppet::Application.new(:filebucket) do @@ -70,10 +70,10 @@ Puppet::Application.new(:filebucket) do begin if options[:local] or options[:bucket] path = options[:bucket] || Puppet[:bucketdir] - @client = Puppet::Network::Client.dipper.new(:Path => path) + @client = Puppet::FileBucket::Dipper.new(:Path => path) else require 'puppet/network/handler' - @client = Puppet::Network::Client.dipper.new(:Server => Puppet[:server]) + @client = Puppet::FileBucket::Dipper.new(:Server => Puppet[:server]) end rescue => detail $stderr.puts detail @@ -84,4 +84,5 @@ Puppet::Application.new(:filebucket) do end end -end
\ No newline at end of file +end + diff --git a/lib/puppet/application/server.rb b/lib/puppet/application/server.rb index afdad54fb..7aeb6ad5d 100644 --- a/lib/puppet/application/server.rb +++ b/lib/puppet/application/server.rb @@ -81,7 +81,6 @@ Puppet::Application.new(:server) do require 'etc' require 'puppet/file_serving/content' require 'puppet/file_serving/metadata' - require 'puppet/checksum' xmlrpc_handlers = [:Status, :FileServer, :Master, :Report, :Filebucket] diff --git a/lib/puppet/checksum.rb b/lib/puppet/checksum.rb deleted file mode 100644 index 27f08aa99..000000000 --- a/lib/puppet/checksum.rb +++ /dev/null @@ -1,57 +0,0 @@ -# -# Created by Luke Kanies on 2007-9-22. -# Copyright (c) 2007. All rights reserved. - -require 'puppet' -require 'puppet/util/checksums' -require 'puppet/indirector' - -# A checksum class to model translating checksums to file paths. This -# is the new filebucket. -class Puppet::Checksum - include Puppet::Util::Checksums - - extend Puppet::Indirector - - indirects :checksum - - attr_reader :algorithm, :content - - def algorithm=(value) - unless respond_to?(value) - raise ArgumentError, "Checksum algorithm %s is not supported" % value - end - value = value.intern if value.is_a?(String) - @algorithm = value - # Reset the checksum so it's forced to be recalculated. - @checksum = nil - end - - # Calculate (if necessary) and return the checksum - def checksum - unless @checksum - @checksum = send(algorithm, content) - end - @checksum - end - - def initialize(content, algorithm = "md5") - raise ArgumentError.new("You must specify the content") unless content - - @content = content - - # Init to avoid warnings. - @checksum = nil - - self.algorithm = algorithm - end - - # This is here so the Indirector::File terminus works correctly. - def name - checksum - end - - def to_s - "Checksum<{%s}%s>" % [algorithm, checksum] - end -end diff --git a/lib/puppet/file_bucket.rb b/lib/puppet/file_bucket.rb new file mode 100644 index 000000000..881c81e58 --- /dev/null +++ b/lib/puppet/file_bucket.rb @@ -0,0 +1,4 @@ +# stub +module Puppet::FileBucket + class BucketError < RuntimeError; end +end diff --git a/lib/puppet/network/client/dipper.rb b/lib/puppet/file_bucket/dipper.rb index 0e2dc1425..c73d76345 100644 --- a/lib/puppet/network/client/dipper.rb +++ b/lib/puppet/file_bucket/dipper.rb @@ -1,32 +1,47 @@ -# The client class for filebuckets. -class Puppet::Network::Client::Dipper < Puppet::Network::Client - @handler = Puppet::Network::Handler.handler(:filebucket) - @drivername = :Bucket +require 'puppet/file_bucket' +require 'puppet/file_bucket/file' +require 'puppet/indirector/request' + +class Puppet::FileBucket::Dipper + # This is a transitional implementation that uses REST + # to access remote filebucket files. attr_accessor :name # Create our bucket client def initialize(hash = {}) + # Emulate the XMLRPC client + server = hash[:Server] + port = hash[:Port] || Puppet[:masterport] + environment = Puppet[:environment] + if hash.include?(:Path) - bucket = self.class.handler.new(:Path => hash[:Path]) - hash.delete(:Path) - hash[:Bucket] = bucket + @local_path = hash[:Path] + @rest_path = nil + else + @local_path = nil + @rest_path = "https://#{server}:#{port}/#{environment}/file_bucket_file/" end + end - super(hash) + def local? + !! @local_path end # Back up a file to our bucket def backup(file) - unless FileTest.exists?(file) + unless ::File.exist?(file) raise(ArgumentError, "File %s does not exist" % file) end contents = ::File.read(file) - unless local? - contents = Base64.encode64(contents) - end begin - return @driver.addfile(contents,file) + file_bucket_file = Puppet::FileBucket::File.new(contents, :bucket_path => @local_path, :path => file) + dest_path = "#{@rest_path}#{file_bucket_file.name}" + + request = Puppet::Indirector::Request.new(:file_bucket_file, :save, dest_path) + + file_bucket_file.save(request) + return file_bucket_file.checksum_data rescue => detail puts detail.backtrace if Puppet[:trace] raise Puppet::Error, "Could not back up %s: %s" % [file, detail] @@ -35,13 +50,10 @@ class Puppet::Network::Client::Dipper < Puppet::Network::Client # Retrieve a file by sum. def getfile(sum) - if newcontents = @driver.getfile(sum) - unless local? - newcontents = Base64.decode64(newcontents) - end - return newcontents - end - return nil + source_path = "#{@rest_path}md5/#{sum}" + file_bucket_file = Puppet::FileBucket::File.find(source_path) + + return file_bucket_file.to_s end # Restore the file diff --git a/lib/puppet/file_bucket/file.rb b/lib/puppet/file_bucket/file.rb new file mode 100644 index 000000000..7122bfbbb --- /dev/null +++ b/lib/puppet/file_bucket/file.rb @@ -0,0 +1,123 @@ +require 'puppet/file_bucket' +require 'puppet/indirector' + +class Puppet::FileBucket::File + # This class handles the abstract notion of a file in a filebucket. + # There are mechanisms to save and load this file locally and remotely in puppet/indirector/filebucketfile/* + # There is a compatibility class that emulates pre-indirector filebuckets in Puppet::FileBucket::Dipper + extend Puppet::Indirector + require 'puppet/file_bucket/file/indirection_hooks' + indirects :file_bucket_file, :terminus_class => :file, :extend => Puppet::FileBucket::File::IndirectionHooks + + attr :path, true + attr :paths, true + attr :contents, true + attr :checksum_type + attr :bucket_path + + def self.default_checksum_type + :md5 + end + + def initialize( contents, options = {} ) + @contents = contents + @bucket_path = options[:bucket_path] + @path = options[:path] + @paths = options[:paths] || [] + @checksum = options[:checksum] + @checksum_type = options[:checksum_type] || self.class.default_checksum_type + + yield(self) if block_given? + + validate! + end + + def validate! + digest_class( @checksum_type ) # raises error on bad types + raise ArgumentError, 'contents must be a string' unless @contents.is_a?(String) + validate_checksum(@checksum) if @checksum + end + + def contents=(contents) + raise "You may not change the contents of a FileBucket File" if @contents + @contents = contents + end + + def checksum=(checksum) + validate_checksum(checksum) + self.checksum_type = checksum # this grabs the prefix only + @checksum = checksum + end + + def validate_checksum(new_checksum) + unless new_checksum == checksum_of_type(new_checksum) + raise Puppet::Error, "checksum does not match contents" + end + end + + def checksum + @checksum ||= checksum_of_type(checksum_type) + end + + def checksum_of_type( type ) + type = checksum_type( type ) # strip out data segment if there is one + type.to_s + ":" + digest_class(type).hexdigest(@contents) + end + + def checksum_type=( new_checksum_type ) + @checksum = nil + @checksum_type = checksum_type(new_checksum_type) + end + + def checksum_type(checksum = @checksum_type) + checksum.to_s.split(':',2)[0].to_sym + end + + def checksum_data(new_checksum = self.checksum) + new_checksum.split(':',2)[1] + end + + def checksum_data=(new_data) + self.checksum = "#{checksum_type}:#{new_data}" + end + + def digest_class(type = nil) + case checksum_type(type) + when :md5 : require 'digest/md5' ; Digest::MD5 + when :sha1 : require 'digest/sha1' ; Digest::SHA1 + else + raise ArgumentError, "not a known checksum type: #{checksum_type(type)}" + end + end + + def to_s + contents + end + + def name + [checksum_type, checksum_data, path].compact.join('/') + end + + def name=(name) + self.checksum_type, self.checksum_data, self.path = name.split('/',3) + end + + def conflict_check? + true + end + + def self.from_s( contents ) + self.new( contents ) + end + + def to_pson + hash = { "contents" => contents } + hash["path"] = @path if @path + hash.to_pson + end + + def self.from_pson( pson ) + self.new( pson["contents"], :path => pson["path"] ) + end + +end diff --git a/lib/puppet/file_bucket/file/indirection_hooks.rb b/lib/puppet/file_bucket/file/indirection_hooks.rb new file mode 100644 index 000000000..ec2bb3469 --- /dev/null +++ b/lib/puppet/file_bucket/file/indirection_hooks.rb @@ -0,0 +1,10 @@ +require 'puppet/file_bucket/file' + +# This module is used to pick the appropriate terminus +# in filebucket indirections. +module Puppet::FileBucket::File::IndirectionHooks + def select_terminus(request) + return :rest if request.protocol == 'https' + return Puppet::FileBucket::File.indirection.terminus_class + end +end diff --git a/lib/puppet/indirector/file_bucket_file/file.rb b/lib/puppet/indirector/file_bucket_file/file.rb new file mode 100644 index 000000000..ec02ca0bd --- /dev/null +++ b/lib/puppet/indirector/file_bucket_file/file.rb @@ -0,0 +1,142 @@ +require 'puppet/indirector/code' +require 'puppet/file_bucket/file' + +module Puppet::FileBucketFile + class File < Puppet::Indirector::Code + desc "Store files in a directory set based on their checksums." + + def initialize + Puppet.settings.use(:filebucket) + end + + def find( request ) + checksum, path = request_to_checksum_and_path( request ) + return find_by_checksum( checksum ) + end + + def save( request ) + checksum, path = request_to_checksum_and_path( request ) + + instance = request.instance + instance.checksum = checksum if checksum + instance.path = path if path + + save_to_disk(instance) + instance.to_s + end + + private + + def find_by_checksum( checksum ) + model.new( nil, :checksum => checksum ) do |bucket_file| + filename = contents_path_for bucket_file + + if ! ::File.exist? filename + return nil + end + + begin + contents = ::File.read filename + Puppet.info "FileBucket read #{bucket_file.checksum}" + rescue RuntimeError => e + raise Puppet::Error, "file could not be read: #{e.message}" + end + + if ::File.exist?(paths_path_for bucket_file) + ::File.open(paths_path_for bucket_file) do |f| + bucket_file.paths = f.readlines.map { |l| l.chomp } + end + end + + bucket_file.contents = contents + end + end + + def save_to_disk( bucket_file ) + # If the file already exists, just return the md5 sum. + if ::File.exist?(contents_path_for bucket_file) + verify_identical_file!(bucket_file) + else + # Make the directories if necessary. + unless ::File.directory?(path_for bucket_file) + Puppet::Util.withumask(0007) do + ::FileUtils.mkdir_p(path_for bucket_file) + end + end + + Puppet.info "FileBucket adding #{bucket_file.path} (#{bucket_file.checksum_data})" + + # Write the file to disk. + Puppet::Util.withumask(0007) do + ::File.open(contents_path_for(bucket_file), ::File::WRONLY|::File::CREAT, 0440) do |of| + of.print bucket_file.contents + end + end + end + + save_path_to_paths_file(bucket_file) + return bucket_file.checksum_data + end + + def request_to_checksum_and_path( request ) + checksum_type, checksum, path = request.key.split(/[:\/]/, 3) + return nil if checksum_type.to_s == "" + return [ checksum_type + ":" + checksum, path ] + end + + def path_for(bucket_file, subfile = nil) + bucket_path = bucket_file.bucket_path || Puppet[:bucketdir] + digest = bucket_file.checksum_data + + dir = ::File.join(digest[0..7].split("")) + basedir = ::File.join(bucket_path, dir, digest) + + return basedir unless subfile + return ::File.join(basedir, subfile) + end + + def contents_path_for(bucket_file) + path_for(bucket_file, "contents") + end + + def paths_path_for(bucket_file) + path_for(bucket_file, "paths") + end + + def content_check? + true + end + + # If conflict_check is enabled, verify that the passed text is + # the same as the text in our file. + def verify_identical_file!(bucket_file) + return unless content_check? + disk_contents = ::File.read(contents_path_for(bucket_file)) + + # If the contents don't match, then we've found a conflict. + # Unlikely, but quite bad. + if disk_contents != bucket_file.contents + raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}", caller + else + Puppet.info "FileBucket got a duplicate file #{bucket_file.path} (#{bucket_file.checksum})" + end + end + + def save_path_to_paths_file(bucket_file) + return unless bucket_file.path + + # check for dupes + if ::File.exist?(paths_path_for bucket_file) + ::File.open(paths_path_for bucket_file) do |f| + return if f.readlines.collect { |l| l.chomp }.include?(bucket_file.path) + end + end + + # if it's a new file, or if our path isn't in the file yet, add it + ::File.open(paths_path_for(bucket_file), ::File::WRONLY|::File::CREAT|::File::APPEND) do |of| + of.puts bucket_file.path + end + end + + end +end diff --git a/lib/puppet/indirector/file_bucket_file/rest.rb b/lib/puppet/indirector/file_bucket_file/rest.rb new file mode 100644 index 000000000..15e4f331d --- /dev/null +++ b/lib/puppet/indirector/file_bucket_file/rest.rb @@ -0,0 +1,8 @@ +require 'puppet/indirector/rest' +require 'puppet/file_bucket/file' + +module Puppet::FileBucketFile + class Rest < Puppet::Indirector::REST + desc "This is a REST based mechanism to send/retrieve file to/from the filebucket" + end +end diff --git a/lib/puppet/indirector/indirection.rb b/lib/puppet/indirector/indirection.rb index d762701f5..3c6414624 100644 --- a/lib/puppet/indirector/indirection.rb +++ b/lib/puppet/indirector/indirection.rb @@ -116,8 +116,14 @@ class Puppet::Indirector::Indirection end # Set up our request object. - def request(method, key, arguments = nil) - Puppet::Indirector::Request.new(self.name, method, key, arguments) + def request(method, instance_or_key, request_or_options = {}) + if request_or_options.is_a? Puppet::Indirector::Request + request = request_or_options + request.instance = instance_or_key + request.method = method + return request + end + Puppet::Indirector::Request.new(self.name, method, instance_or_key, request_or_options) end # Return the singleton terminus for this indirection. @@ -248,8 +254,8 @@ class Puppet::Indirector::Indirection # Save the instance in the appropriate terminus. This method is # normally an instance method on the indirected class. - def save(instance, *args) - request = request(:save, instance, *args) + def save(instance, request_or_options = nil) + request = request(:save, instance, request_or_options) terminus = prepare(request) result = terminus.save(request) diff --git a/lib/puppet/indirector/request.rb b/lib/puppet/indirector/request.rb index d9e66cb5b..14608d0dc 100644 --- a/lib/puppet/indirector/request.rb +++ b/lib/puppet/indirector/request.rb @@ -180,6 +180,14 @@ class Puppet::Indirector::Request end @protocol = uri.scheme - @key = URI.unescape(uri.path.sub(/^\//, '')) + + if uri.scheme == 'puppet' + @key = URI.unescape(uri.path.sub(/^\//, '')) + return + end + + env, indirector, @key = URI.unescape(uri.path.sub(/^\//, '')).split('/',3) + @key ||= '' + self.environment = env unless env == '' end end diff --git a/lib/puppet/network/handler/filebucket.rb b/lib/puppet/network/handler/filebucket.rb index 4973886f7..bea1c85f5 100755 --- a/lib/puppet/network/handler/filebucket.rb +++ b/lib/puppet/network/handler/filebucket.rb @@ -3,7 +3,6 @@ require 'digest/md5' require 'puppet/external/base64' class Puppet::Network::Handler # :nodoc: - class BucketError < RuntimeError; end # Accept files and store them by md5 sum, returning the md5 sum back # to the client. Alternatively, accept an md5 sum and return the # associated content. @@ -19,53 +18,8 @@ class Puppet::Network::Handler # :nodoc: Puppet::Util.logmethods(self, true) attr_reader :name, :path - # this doesn't work for relative paths - def self.oldpaths(base,md5) - return [ - File.join(base, md5), - File.join(base, md5, "contents"), - File.join(base, md5, "paths") - ] - end - - # this doesn't work for relative paths - def self.paths(base,md5) - dir = File.join(md5[0..7].split("")) - basedir = File.join(base, dir, md5) - return [ - basedir, - File.join(basedir, "contents"), - File.join(basedir, "paths") - ] - end - - # Should we check each file as it comes in to make sure the md5 - # sums match? Defaults to false. - def conflict_check? - @confictchk - end - def initialize(hash) - if hash.include?(:ConflictCheck) - @conflictchk = hash[:ConflictCheck] - hash.delete(:ConflictCheck) - else - @conflictchk = false - end - - if hash.include?(:Path) - @path = hash[:Path] - hash.delete(:Path) - else - if defined? Puppet - @path = Puppet[:bucketdir] - else - @path = File.expand_path("~/.filebucket") - end - end - - Puppet.settings.use(:filebucket) - + @path = hash[:Path] || Puppet[:bucketdir] @name = "Filebucket[#{@path}]" end @@ -75,60 +29,14 @@ class Puppet::Network::Handler # :nodoc: if client contents = Base64.decode64(contents) end - md5 = Digest::MD5.hexdigest(contents) - - bpath, bfile, pathpath = FileBucket.paths(@path,md5) - - # If the file already exists, just return the md5 sum. - if FileTest.exists?(bfile) - # If verification is enabled, then make sure the text matches. - if conflict_check? - verify(contents, md5, bfile) - end - return md5 - end - - # Make the directories if necessary. - unless FileTest.directory?(bpath) - Puppet::Util.withumask(0007) do - FileUtils.mkdir_p(bpath) - end - end - - # Write the file to disk. - msg = "Adding %s(%s)" % [path, md5] - msg += " from #{client}" if client - self.info msg - - # ...then just create the file - Puppet::Util.withumask(0007) do - File.open(bfile, File::WRONLY|File::CREAT, 0440) { |of| - of.print contents - } - end - - # Write the path to the paths file. - add_path(path, pathpath) - - return md5 + bucket = Puppet::FileBucket::File.new(contents) + return bucket.save end # Return the contents associated with a given md5 sum. def getfile(md5, client = nil, clientip = nil) - bpath, bfile, bpaths = FileBucket.paths(@path,md5) - - unless FileTest.exists?(bfile) - # Try the old flat style. - bpath, bfile, bpaths = FileBucket.oldpaths(@path,md5) - unless FileTest.exists?(bfile) - return false - end - end - - contents = nil - File.open(bfile) { |of| - contents = of.read - } + bucket = Puppet::FileBucket::File.find("md5:#{md5}") + contents = bucket.contents if client return Base64.encode64(contents) @@ -137,46 +45,9 @@ class Puppet::Network::Handler # :nodoc: end end - def paths(md5) - self.class(@path, md5) - end - def to_s self.name end - - private - - # Add our path to the paths file if necessary. - def add_path(path, file) - if FileTest.exists?(file) - File.open(file) { |of| - return if of.readlines.collect { |l| l.chomp }.include?(path) - } - end - - # if it's a new file, or if our path isn't in the file yet, add it - File.open(file, File::WRONLY|File::CREAT|File::APPEND) { |of| - of.puts path - } - end - - # If conflict_check is enabled, verify that the passed text is - # the same as the text in our file. - def verify(content, md5, bfile) - curfile = File.read(bfile) - - # If the contents don't match, then we've found a conflict. - # Unlikely, but quite bad. - if curfile != contents - raise(BucketError, - "Got passed new contents for sum %s" % md5, caller) - else - msg = "Got duplicate %s(%s)" % [path, md5] - msg += " from #{client}" if client - self.info msg - end - end end end diff --git a/lib/puppet/network/http/handler.rb b/lib/puppet/network/http/handler.rb index 444fbf7e7..01ca65023 100644 --- a/lib/puppet/network/http/handler.rb +++ b/lib/puppet/network/http/handler.rb @@ -165,7 +165,7 @@ module Puppet::Network::HTTP::Handler # LAK:NOTE This has to be here for testing; it's a stub-point so # we keep infinite recursion from happening. def save_object(ind_request, object) - object.save(ind_request.to_hash) + object.save(ind_request) end def get?(request) diff --git a/lib/puppet/type/filebucket.rb b/lib/puppet/type/filebucket.rb index 468f92638..4754e9f3a 100755 --- a/lib/puppet/type/filebucket.rb +++ b/lib/puppet/type/filebucket.rb @@ -1,5 +1,5 @@ module Puppet - require 'puppet/network/client' + require 'puppet/file_bucket/dipper' newtype(:filebucket) do @doc = "A repository for backing up files. If no filebucket is @@ -83,7 +83,7 @@ module Puppet end begin - @bucket = Puppet::Network::Client.client(:Dipper).new(args) + @bucket = Puppet::FileBucket::Dipper.new(args) rescue => detail puts detail.backtrace if Puppet[:trace] self.fail("Could not create %s filebucket: %s" % [type, detail]) diff --git a/lib/puppet/type/tidy.rb b/lib/puppet/type/tidy.rb index 3d7190c27..830f47640 100755 --- a/lib/puppet/type/tidy.rb +++ b/lib/puppet/type/tidy.rb @@ -215,7 +215,7 @@ Puppet::Type.newtype(:tidy) do super # only allow backing up into filebuckets - unless self[:backup].is_a? Puppet::Network::Client.dipper + unless self[:backup].is_a? Puppet::FileBucket::Dipper self[:backup] = false end end |