diff options
35 files changed, 1078 insertions, 706 deletions
diff --git a/ext/nagios/naggen b/ext/nagios/naggen index 6ff09e260..a9e04c47a 100755 --- a/ext/nagios/naggen +++ b/ext/nagios/naggen @@ -178,7 +178,7 @@ class NagiosWriter def initialize(nagios_type) @nagios_type = nagios_type - @bucket = Puppet::Network::Client.client(:Dipper).new(:Path => Puppet[:clientbucketdir]) + @bucket = Puppet::FileBucket::Dipper.new(:Path => Puppet[:clientbucketdir]) end def rails_resources diff --git a/ext/puppet-test b/ext/puppet-test index dbbde40c6..53333076f 100755 --- a/ext/puppet-test +++ b/ext/puppet-test @@ -286,7 +286,7 @@ end Suite.new :filebucket, "Filebucket interactions" do def prepare require 'tempfile' - @client = Puppet::Network::Client.dipper.new($args) + @client = Puppet::FileBucket::Dipper.new($args) end newtest :backup, "Backed up file" do diff --git a/lib/puppet.rb b/lib/puppet.rb index 1fa51c3b4..0904f044a 100644 --- a/lib/puppet.rb +++ b/lib/puppet.rb @@ -157,6 +157,7 @@ require 'puppet/ssl' require 'puppet/module' require 'puppet/util/storage' require 'puppet/status' +require 'puppet/file_bucket/file' if Puppet[:storeconfigs] require 'puppet/rails' diff --git a/lib/puppet/application/filebucket.rb b/lib/puppet/application/filebucket.rb index 0723054df..09aaf2b5d 100644 --- a/lib/puppet/application/filebucket.rb +++ b/lib/puppet/application/filebucket.rb @@ -1,6 +1,6 @@ require 'puppet' require 'puppet/application' -require 'puppet/network/client' +require 'puppet/file_bucket/dipper' Puppet::Application.new(:filebucket) do @@ -70,10 +70,10 @@ Puppet::Application.new(:filebucket) do begin if options[:local] or options[:bucket] path = options[:bucket] || Puppet[:bucketdir] - @client = Puppet::Network::Client.dipper.new(:Path => path) + @client = Puppet::FileBucket::Dipper.new(:Path => path) else require 'puppet/network/handler' - @client = Puppet::Network::Client.dipper.new(:Server => Puppet[:server]) + @client = Puppet::FileBucket::Dipper.new(:Server => Puppet[:server]) end rescue => detail $stderr.puts detail @@ -84,4 +84,5 @@ Puppet::Application.new(:filebucket) do end end -end
\ No newline at end of file +end + diff --git a/lib/puppet/application/server.rb b/lib/puppet/application/server.rb index afdad54fb..7aeb6ad5d 100644 --- a/lib/puppet/application/server.rb +++ b/lib/puppet/application/server.rb @@ -81,7 +81,6 @@ Puppet::Application.new(:server) do require 'etc' require 'puppet/file_serving/content' require 'puppet/file_serving/metadata' - require 'puppet/checksum' xmlrpc_handlers = [:Status, :FileServer, :Master, :Report, :Filebucket] diff --git a/lib/puppet/checksum.rb b/lib/puppet/checksum.rb deleted file mode 100644 index 27f08aa99..000000000 --- a/lib/puppet/checksum.rb +++ /dev/null @@ -1,57 +0,0 @@ -# -# Created by Luke Kanies on 2007-9-22. -# Copyright (c) 2007. All rights reserved. - -require 'puppet' -require 'puppet/util/checksums' -require 'puppet/indirector' - -# A checksum class to model translating checksums to file paths. This -# is the new filebucket. -class Puppet::Checksum - include Puppet::Util::Checksums - - extend Puppet::Indirector - - indirects :checksum - - attr_reader :algorithm, :content - - def algorithm=(value) - unless respond_to?(value) - raise ArgumentError, "Checksum algorithm %s is not supported" % value - end - value = value.intern if value.is_a?(String) - @algorithm = value - # Reset the checksum so it's forced to be recalculated. - @checksum = nil - end - - # Calculate (if necessary) and return the checksum - def checksum - unless @checksum - @checksum = send(algorithm, content) - end - @checksum - end - - def initialize(content, algorithm = "md5") - raise ArgumentError.new("You must specify the content") unless content - - @content = content - - # Init to avoid warnings. - @checksum = nil - - self.algorithm = algorithm - end - - # This is here so the Indirector::File terminus works correctly. - def name - checksum - end - - def to_s - "Checksum<{%s}%s>" % [algorithm, checksum] - end -end diff --git a/lib/puppet/file_bucket.rb b/lib/puppet/file_bucket.rb new file mode 100644 index 000000000..881c81e58 --- /dev/null +++ b/lib/puppet/file_bucket.rb @@ -0,0 +1,4 @@ +# stub +module Puppet::FileBucket + class BucketError < RuntimeError; end +end diff --git a/lib/puppet/network/client/dipper.rb b/lib/puppet/file_bucket/dipper.rb index 0e2dc1425..c73d76345 100644 --- a/lib/puppet/network/client/dipper.rb +++ b/lib/puppet/file_bucket/dipper.rb @@ -1,32 +1,47 @@ -# The client class for filebuckets. -class Puppet::Network::Client::Dipper < Puppet::Network::Client - @handler = Puppet::Network::Handler.handler(:filebucket) - @drivername = :Bucket +require 'puppet/file_bucket' +require 'puppet/file_bucket/file' +require 'puppet/indirector/request' + +class Puppet::FileBucket::Dipper + # This is a transitional implementation that uses REST + # to access remote filebucket files. attr_accessor :name # Create our bucket client def initialize(hash = {}) + # Emulate the XMLRPC client + server = hash[:Server] + port = hash[:Port] || Puppet[:masterport] + environment = Puppet[:environment] + if hash.include?(:Path) - bucket = self.class.handler.new(:Path => hash[:Path]) - hash.delete(:Path) - hash[:Bucket] = bucket + @local_path = hash[:Path] + @rest_path = nil + else + @local_path = nil + @rest_path = "https://#{server}:#{port}/#{environment}/file_bucket_file/" end + end - super(hash) + def local? + !! @local_path end # Back up a file to our bucket def backup(file) - unless FileTest.exists?(file) + unless ::File.exist?(file) raise(ArgumentError, "File %s does not exist" % file) end contents = ::File.read(file) - unless local? - contents = Base64.encode64(contents) - end begin - return @driver.addfile(contents,file) + file_bucket_file = Puppet::FileBucket::File.new(contents, :bucket_path => @local_path, :path => file) + dest_path = "#{@rest_path}#{file_bucket_file.name}" + + request = Puppet::Indirector::Request.new(:file_bucket_file, :save, dest_path) + + file_bucket_file.save(request) + return file_bucket_file.checksum_data rescue => detail puts detail.backtrace if Puppet[:trace] raise Puppet::Error, "Could not back up %s: %s" % [file, detail] @@ -35,13 +50,10 @@ class Puppet::Network::Client::Dipper < Puppet::Network::Client # Retrieve a file by sum. def getfile(sum) - if newcontents = @driver.getfile(sum) - unless local? - newcontents = Base64.decode64(newcontents) - end - return newcontents - end - return nil + source_path = "#{@rest_path}md5/#{sum}" + file_bucket_file = Puppet::FileBucket::File.find(source_path) + + return file_bucket_file.to_s end # Restore the file diff --git a/lib/puppet/file_bucket/file.rb b/lib/puppet/file_bucket/file.rb new file mode 100644 index 000000000..7122bfbbb --- /dev/null +++ b/lib/puppet/file_bucket/file.rb @@ -0,0 +1,123 @@ +require 'puppet/file_bucket' +require 'puppet/indirector' + +class Puppet::FileBucket::File + # This class handles the abstract notion of a file in a filebucket. + # There are mechanisms to save and load this file locally and remotely in puppet/indirector/filebucketfile/* + # There is a compatibility class that emulates pre-indirector filebuckets in Puppet::FileBucket::Dipper + extend Puppet::Indirector + require 'puppet/file_bucket/file/indirection_hooks' + indirects :file_bucket_file, :terminus_class => :file, :extend => Puppet::FileBucket::File::IndirectionHooks + + attr :path, true + attr :paths, true + attr :contents, true + attr :checksum_type + attr :bucket_path + + def self.default_checksum_type + :md5 + end + + def initialize( contents, options = {} ) + @contents = contents + @bucket_path = options[:bucket_path] + @path = options[:path] + @paths = options[:paths] || [] + @checksum = options[:checksum] + @checksum_type = options[:checksum_type] || self.class.default_checksum_type + + yield(self) if block_given? + + validate! + end + + def validate! + digest_class( @checksum_type ) # raises error on bad types + raise ArgumentError, 'contents must be a string' unless @contents.is_a?(String) + validate_checksum(@checksum) if @checksum + end + + def contents=(contents) + raise "You may not change the contents of a FileBucket File" if @contents + @contents = contents + end + + def checksum=(checksum) + validate_checksum(checksum) + self.checksum_type = checksum # this grabs the prefix only + @checksum = checksum + end + + def validate_checksum(new_checksum) + unless new_checksum == checksum_of_type(new_checksum) + raise Puppet::Error, "checksum does not match contents" + end + end + + def checksum + @checksum ||= checksum_of_type(checksum_type) + end + + def checksum_of_type( type ) + type = checksum_type( type ) # strip out data segment if there is one + type.to_s + ":" + digest_class(type).hexdigest(@contents) + end + + def checksum_type=( new_checksum_type ) + @checksum = nil + @checksum_type = checksum_type(new_checksum_type) + end + + def checksum_type(checksum = @checksum_type) + checksum.to_s.split(':',2)[0].to_sym + end + + def checksum_data(new_checksum = self.checksum) + new_checksum.split(':',2)[1] + end + + def checksum_data=(new_data) + self.checksum = "#{checksum_type}:#{new_data}" + end + + def digest_class(type = nil) + case checksum_type(type) + when :md5 : require 'digest/md5' ; Digest::MD5 + when :sha1 : require 'digest/sha1' ; Digest::SHA1 + else + raise ArgumentError, "not a known checksum type: #{checksum_type(type)}" + end + end + + def to_s + contents + end + + def name + [checksum_type, checksum_data, path].compact.join('/') + end + + def name=(name) + self.checksum_type, self.checksum_data, self.path = name.split('/',3) + end + + def conflict_check? + true + end + + def self.from_s( contents ) + self.new( contents ) + end + + def to_pson + hash = { "contents" => contents } + hash["path"] = @path if @path + hash.to_pson + end + + def self.from_pson( pson ) + self.new( pson["contents"], :path => pson["path"] ) + end + +end diff --git a/lib/puppet/file_bucket/file/indirection_hooks.rb b/lib/puppet/file_bucket/file/indirection_hooks.rb new file mode 100644 index 000000000..ec2bb3469 --- /dev/null +++ b/lib/puppet/file_bucket/file/indirection_hooks.rb @@ -0,0 +1,10 @@ +require 'puppet/file_bucket/file' + +# This module is used to pick the appropriate terminus +# in filebucket indirections. +module Puppet::FileBucket::File::IndirectionHooks + def select_terminus(request) + return :rest if request.protocol == 'https' + return Puppet::FileBucket::File.indirection.terminus_class + end +end diff --git a/lib/puppet/indirector/file_bucket_file/file.rb b/lib/puppet/indirector/file_bucket_file/file.rb new file mode 100644 index 000000000..ec02ca0bd --- /dev/null +++ b/lib/puppet/indirector/file_bucket_file/file.rb @@ -0,0 +1,142 @@ +require 'puppet/indirector/code' +require 'puppet/file_bucket/file' + +module Puppet::FileBucketFile + class File < Puppet::Indirector::Code + desc "Store files in a directory set based on their checksums." + + def initialize + Puppet.settings.use(:filebucket) + end + + def find( request ) + checksum, path = request_to_checksum_and_path( request ) + return find_by_checksum( checksum ) + end + + def save( request ) + checksum, path = request_to_checksum_and_path( request ) + + instance = request.instance + instance.checksum = checksum if checksum + instance.path = path if path + + save_to_disk(instance) + instance.to_s + end + + private + + def find_by_checksum( checksum ) + model.new( nil, :checksum => checksum ) do |bucket_file| + filename = contents_path_for bucket_file + + if ! ::File.exist? filename + return nil + end + + begin + contents = ::File.read filename + Puppet.info "FileBucket read #{bucket_file.checksum}" + rescue RuntimeError => e + raise Puppet::Error, "file could not be read: #{e.message}" + end + + if ::File.exist?(paths_path_for bucket_file) + ::File.open(paths_path_for bucket_file) do |f| + bucket_file.paths = f.readlines.map { |l| l.chomp } + end + end + + bucket_file.contents = contents + end + end + + def save_to_disk( bucket_file ) + # If the file already exists, just return the md5 sum. + if ::File.exist?(contents_path_for bucket_file) + verify_identical_file!(bucket_file) + else + # Make the directories if necessary. + unless ::File.directory?(path_for bucket_file) + Puppet::Util.withumask(0007) do + ::FileUtils.mkdir_p(path_for bucket_file) + end + end + + Puppet.info "FileBucket adding #{bucket_file.path} (#{bucket_file.checksum_data})" + + # Write the file to disk. + Puppet::Util.withumask(0007) do + ::File.open(contents_path_for(bucket_file), ::File::WRONLY|::File::CREAT, 0440) do |of| + of.print bucket_file.contents + end + end + end + + save_path_to_paths_file(bucket_file) + return bucket_file.checksum_data + end + + def request_to_checksum_and_path( request ) + checksum_type, checksum, path = request.key.split(/[:\/]/, 3) + return nil if checksum_type.to_s == "" + return [ checksum_type + ":" + checksum, path ] + end + + def path_for(bucket_file, subfile = nil) + bucket_path = bucket_file.bucket_path || Puppet[:bucketdir] + digest = bucket_file.checksum_data + + dir = ::File.join(digest[0..7].split("")) + basedir = ::File.join(bucket_path, dir, digest) + + return basedir unless subfile + return ::File.join(basedir, subfile) + end + + def contents_path_for(bucket_file) + path_for(bucket_file, "contents") + end + + def paths_path_for(bucket_file) + path_for(bucket_file, "paths") + end + + def content_check? + true + end + + # If conflict_check is enabled, verify that the passed text is + # the same as the text in our file. + def verify_identical_file!(bucket_file) + return unless content_check? + disk_contents = ::File.read(contents_path_for(bucket_file)) + + # If the contents don't match, then we've found a conflict. + # Unlikely, but quite bad. + if disk_contents != bucket_file.contents + raise Puppet::FileBucket::BucketError, "Got passed new contents for sum #{bucket_file.checksum}", caller + else + Puppet.info "FileBucket got a duplicate file #{bucket_file.path} (#{bucket_file.checksum})" + end + end + + def save_path_to_paths_file(bucket_file) + return unless bucket_file.path + + # check for dupes + if ::File.exist?(paths_path_for bucket_file) + ::File.open(paths_path_for bucket_file) do |f| + return if f.readlines.collect { |l| l.chomp }.include?(bucket_file.path) + end + end + + # if it's a new file, or if our path isn't in the file yet, add it + ::File.open(paths_path_for(bucket_file), ::File::WRONLY|::File::CREAT|::File::APPEND) do |of| + of.puts bucket_file.path + end + end + + end +end diff --git a/lib/puppet/indirector/file_bucket_file/rest.rb b/lib/puppet/indirector/file_bucket_file/rest.rb new file mode 100644 index 000000000..15e4f331d --- /dev/null +++ b/lib/puppet/indirector/file_bucket_file/rest.rb @@ -0,0 +1,8 @@ +require 'puppet/indirector/rest' +require 'puppet/file_bucket/file' + +module Puppet::FileBucketFile + class Rest < Puppet::Indirector::REST + desc "This is a REST based mechanism to send/retrieve file to/from the filebucket" + end +end diff --git a/lib/puppet/indirector/indirection.rb b/lib/puppet/indirector/indirection.rb index d762701f5..3c6414624 100644 --- a/lib/puppet/indirector/indirection.rb +++ b/lib/puppet/indirector/indirection.rb @@ -116,8 +116,14 @@ class Puppet::Indirector::Indirection end # Set up our request object. - def request(method, key, arguments = nil) - Puppet::Indirector::Request.new(self.name, method, key, arguments) + def request(method, instance_or_key, request_or_options = {}) + if request_or_options.is_a? Puppet::Indirector::Request + request = request_or_options + request.instance = instance_or_key + request.method = method + return request + end + Puppet::Indirector::Request.new(self.name, method, instance_or_key, request_or_options) end # Return the singleton terminus for this indirection. @@ -248,8 +254,8 @@ class Puppet::Indirector::Indirection # Save the instance in the appropriate terminus. This method is # normally an instance method on the indirected class. - def save(instance, *args) - request = request(:save, instance, *args) + def save(instance, request_or_options = nil) + request = request(:save, instance, request_or_options) terminus = prepare(request) result = terminus.save(request) diff --git a/lib/puppet/indirector/request.rb b/lib/puppet/indirector/request.rb index d9e66cb5b..14608d0dc 100644 --- a/lib/puppet/indirector/request.rb +++ b/lib/puppet/indirector/request.rb @@ -180,6 +180,14 @@ class Puppet::Indirector::Request end @protocol = uri.scheme - @key = URI.unescape(uri.path.sub(/^\//, '')) + + if uri.scheme == 'puppet' + @key = URI.unescape(uri.path.sub(/^\//, '')) + return + end + + env, indirector, @key = URI.unescape(uri.path.sub(/^\//, '')).split('/',3) + @key ||= '' + self.environment = env unless env == '' end end diff --git a/lib/puppet/network/handler/filebucket.rb b/lib/puppet/network/handler/filebucket.rb index 4973886f7..bea1c85f5 100755 --- a/lib/puppet/network/handler/filebucket.rb +++ b/lib/puppet/network/handler/filebucket.rb @@ -3,7 +3,6 @@ require 'digest/md5' require 'puppet/external/base64' class Puppet::Network::Handler # :nodoc: - class BucketError < RuntimeError; end # Accept files and store them by md5 sum, returning the md5 sum back # to the client. Alternatively, accept an md5 sum and return the # associated content. @@ -19,53 +18,8 @@ class Puppet::Network::Handler # :nodoc: Puppet::Util.logmethods(self, true) attr_reader :name, :path - # this doesn't work for relative paths - def self.oldpaths(base,md5) - return [ - File.join(base, md5), - File.join(base, md5, "contents"), - File.join(base, md5, "paths") - ] - end - - # this doesn't work for relative paths - def self.paths(base,md5) - dir = File.join(md5[0..7].split("")) - basedir = File.join(base, dir, md5) - return [ - basedir, - File.join(basedir, "contents"), - File.join(basedir, "paths") - ] - end - - # Should we check each file as it comes in to make sure the md5 - # sums match? Defaults to false. - def conflict_check? - @confictchk - end - def initialize(hash) - if hash.include?(:ConflictCheck) - @conflictchk = hash[:ConflictCheck] - hash.delete(:ConflictCheck) - else - @conflictchk = false - end - - if hash.include?(:Path) - @path = hash[:Path] - hash.delete(:Path) - else - if defined? Puppet - @path = Puppet[:bucketdir] - else - @path = File.expand_path("~/.filebucket") - end - end - - Puppet.settings.use(:filebucket) - + @path = hash[:Path] || Puppet[:bucketdir] @name = "Filebucket[#{@path}]" end @@ -75,60 +29,14 @@ class Puppet::Network::Handler # :nodoc: if client contents = Base64.decode64(contents) end - md5 = Digest::MD5.hexdigest(contents) - - bpath, bfile, pathpath = FileBucket.paths(@path,md5) - - # If the file already exists, just return the md5 sum. - if FileTest.exists?(bfile) - # If verification is enabled, then make sure the text matches. - if conflict_check? - verify(contents, md5, bfile) - end - return md5 - end - - # Make the directories if necessary. - unless FileTest.directory?(bpath) - Puppet::Util.withumask(0007) do - FileUtils.mkdir_p(bpath) - end - end - - # Write the file to disk. - msg = "Adding %s(%s)" % [path, md5] - msg += " from #{client}" if client - self.info msg - - # ...then just create the file - Puppet::Util.withumask(0007) do - File.open(bfile, File::WRONLY|File::CREAT, 0440) { |of| - of.print contents - } - end - - # Write the path to the paths file. - add_path(path, pathpath) - - return md5 + bucket = Puppet::FileBucket::File.new(contents) + return bucket.save end # Return the contents associated with a given md5 sum. def getfile(md5, client = nil, clientip = nil) - bpath, bfile, bpaths = FileBucket.paths(@path,md5) - - unless FileTest.exists?(bfile) - # Try the old flat style. - bpath, bfile, bpaths = FileBucket.oldpaths(@path,md5) - unless FileTest.exists?(bfile) - return false - end - end - - contents = nil - File.open(bfile) { |of| - contents = of.read - } + bucket = Puppet::FileBucket::File.find("md5:#{md5}") + contents = bucket.contents if client return Base64.encode64(contents) @@ -137,46 +45,9 @@ class Puppet::Network::Handler # :nodoc: end end - def paths(md5) - self.class(@path, md5) - end - def to_s self.name end - - private - - # Add our path to the paths file if necessary. - def add_path(path, file) - if FileTest.exists?(file) - File.open(file) { |of| - return if of.readlines.collect { |l| l.chomp }.include?(path) - } - end - - # if it's a new file, or if our path isn't in the file yet, add it - File.open(file, File::WRONLY|File::CREAT|File::APPEND) { |of| - of.puts path - } - end - - # If conflict_check is enabled, verify that the passed text is - # the same as the text in our file. - def verify(content, md5, bfile) - curfile = File.read(bfile) - - # If the contents don't match, then we've found a conflict. - # Unlikely, but quite bad. - if curfile != contents - raise(BucketError, - "Got passed new contents for sum %s" % md5, caller) - else - msg = "Got duplicate %s(%s)" % [path, md5] - msg += " from #{client}" if client - self.info msg - end - end end end diff --git a/lib/puppet/network/http/handler.rb b/lib/puppet/network/http/handler.rb index 444fbf7e7..01ca65023 100644 --- a/lib/puppet/network/http/handler.rb +++ b/lib/puppet/network/http/handler.rb @@ -165,7 +165,7 @@ module Puppet::Network::HTTP::Handler # LAK:NOTE This has to be here for testing; it's a stub-point so # we keep infinite recursion from happening. def save_object(ind_request, object) - object.save(ind_request.to_hash) + object.save(ind_request) end def get?(request) diff --git a/lib/puppet/type/filebucket.rb b/lib/puppet/type/filebucket.rb index 468f92638..4754e9f3a 100755 --- a/lib/puppet/type/filebucket.rb +++ b/lib/puppet/type/filebucket.rb @@ -1,5 +1,5 @@ module Puppet - require 'puppet/network/client' + require 'puppet/file_bucket/dipper' newtype(:filebucket) do @doc = "A repository for backing up files. If no filebucket is @@ -83,7 +83,7 @@ module Puppet end begin - @bucket = Puppet::Network::Client.client(:Dipper).new(args) + @bucket = Puppet::FileBucket::Dipper.new(args) rescue => detail puts detail.backtrace if Puppet[:trace] self.fail("Could not create %s filebucket: %s" % [type, detail]) diff --git a/lib/puppet/type/tidy.rb b/lib/puppet/type/tidy.rb index 3d7190c27..830f47640 100755 --- a/lib/puppet/type/tidy.rb +++ b/lib/puppet/type/tidy.rb @@ -215,7 +215,7 @@ Puppet::Type.newtype(:tidy) do super # only allow backing up into filebuckets - unless self[:backup].is_a? Puppet::Network::Client.dipper + unless self[:backup].is_a? Puppet::FileBucket::Dipper self[:backup] = false end end diff --git a/spec/integration/checksum.rb b/spec/integration/checksum.rb deleted file mode 100755 index 49ae8d2b7..000000000 --- a/spec/integration/checksum.rb +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env ruby -# -# Created by Luke Kanies on 2007-9-22. -# Copyright (c) 2007. All rights reserved. - -require File.dirname(__FILE__) + '/../spec_helper' - -require 'puppet/checksum' - -describe Puppet::Checksum, " when using the file terminus" do - before do - Puppet.settings.stubs(:use) - Puppet::Checksum.terminus_class = :file - @content = "this is some content" - @sum = Puppet::Checksum.new(@content) - - @file = Puppet::Checksum.indirection.terminus.path(@sum.checksum) - end - - it "should store content at a path determined by its checksum" do - File.stubs(:directory?).returns(true) - filehandle = mock 'filehandle' - filehandle.expects(:print).with(@content) - File.expects(:open).with(@file, "w").yields(filehandle) - - @sum.save - end - - it "should retrieve stored content when the checksum is provided as the key" do - File.stubs(:exist?).returns(true) - File.expects(:read).with(@file).returns(@content) - - newsum = Puppet::Checksum.find(@sum.checksum) - - newsum.content.should == @content - end - - it "should remove specified files when asked" do - File.stubs(:exist?).returns(true) - File.expects(:unlink).with(@file) - - Puppet::Checksum.destroy(@sum.name) - end - - after do - Puppet.settings.clear - end -end diff --git a/spec/integration/indirector/bucket_file/rest.rb b/spec/integration/indirector/bucket_file/rest.rb new file mode 100644 index 000000000..296b03eb6 --- /dev/null +++ b/spec/integration/indirector/bucket_file/rest.rb @@ -0,0 +1,68 @@ +#!/usr/bin/env ruby + +Dir.chdir(File.dirname(__FILE__)) { (s = lambda { |f| File.exist?(f) ? require(f) : Dir.chdir("..") { s.call(f) } }).call("spec/spec_helper.rb") } + +require 'puppet/file_bucket/file' +require 'puppet/network/server' +require 'puppet/network/http/webrick/rest' + +describe "Filebucket REST Terminus" do + before do + Puppet[:masterport] = 34343 + Puppet[:server] = "localhost" + + # Get a safe temporary file + @tmpfile = Tempfile.new("webrick_integration_testing") + @dir = @tmpfile.path + "_dir" + + Puppet.settings[:confdir] = @dir + Puppet.settings[:vardir] = @dir + Puppet.settings[:server] = "127.0.0.1" + Puppet.settings[:masterport] = "34343" + + Puppet::Util::Cacher.expire + + Puppet[:servertype] = 'webrick' + Puppet[:server] = '127.0.0.1' + Puppet[:certname] = '127.0.0.1' + + # Generate the certificate with a local CA + Puppet::SSL::Host.ca_location = :local + ca = Puppet::SSL::CertificateAuthority.new + ca.generate(Puppet[:certname]) unless Puppet::SSL::Certificate.find(Puppet[:certname]) + ca.generate("foo.madstop.com") unless Puppet::SSL::Certificate.find(Puppet[:certname]) + + @host = Puppet::SSL::Host.new(Puppet[:certname]) + + @params = { :port => 34343, :handlers => [ :file_bucket_file ] } + @server = Puppet::Network::Server.new(@params) + @server.listen + + @old_terminus = Puppet::FileBucket::File.indirection.terminus_class + Puppet::FileBucket::File.terminus_class = :rest + + # LAK:NOTE We need to have a fake model here so that our indirected methods get + # passed through REST; otherwise we'd be stubbing 'find', which would cause an immediate + # return. + @file_bucket_file = stub_everything 'file_bucket_file' + @mock_model = stub('faked model', :name => "file_bucket_file", :convert_from => @file_bucket_file) + Puppet::Indirector::Request.any_instance.stubs(:model).returns(@mock_model) + + Puppet::Network::HTTP::WEBrickREST.any_instance.stubs(:check_authorization) + end + + after do + Puppet::Network::HttpPool.expire + Puppet::SSL::Host.ca_location = :none + Puppet.settings.clear + @server.unlisten + Puppet::FileBucket::File.terminus_class = @old_terminus + end + + it "should be able save a file to the remote filebucket" do + @file_bucket_file.expects(:save) + + file_bucket_file = Puppet::FileBucket::File.new("pouet") + file_bucket_file.save() + end +end diff --git a/spec/integration/network/client.rb b/spec/integration/network/client.rb index 970763712..fe1524e60 100755 --- a/spec/integration/network/client.rb +++ b/spec/integration/network/client.rb @@ -5,7 +5,7 @@ require File.dirname(__FILE__) + '/../../spec_helper' require 'puppet/network/client' describe Puppet::Network::Client do - %w{ca dipper file report resource runner status}.each do |name| + %w{ca file report resource runner status}.each do |name| it "should have a #{name} client" do Puppet::Network::Client.client(name).should be_instance_of(Class) end diff --git a/spec/unit/application/filebucket.rb b/spec/unit/application/filebucket.rb index e87bab402..f78c0b7be 100644 --- a/spec/unit/application/filebucket.rb +++ b/spec/unit/application/filebucket.rb @@ -43,7 +43,7 @@ describe "Filebucket" do Puppet.stubs(:settraps) Puppet::Log.stubs(:level=) Puppet.stubs(:parse_config) - Puppet::Network::Client.dipper.stubs(:new) + Puppet::FileBucket::Dipper.stubs(:new) @filebucket.options.stubs(:[]).with(any_parameters) end @@ -106,15 +106,15 @@ describe "Filebucket" do it "should create a client with the default bucket if none passed" do Puppet.stubs(:[]).with(:bucketdir).returns("path") - Puppet::Network::Client::Dipper.expects(:new).with { |h| h[:Path] == "path" } + Puppet::FileBucket::Dipper.expects(:new).with { |h| h[:Path] == "path" } @filebucket.run_setup end - it "should create a local Client dipper with the given bucket" do + it "should create a local Dipper with the given bucket" do @filebucket.options.stubs(:[]).with(:bucket).returns("path") - Puppet::Network::Client::Dipper.expects(:new).with { |h| h[:Path] == "path" } + Puppet::FileBucket::Dipper.expects(:new).with { |h| h[:Path] == "path" } @filebucket.run_setup end @@ -126,7 +126,7 @@ describe "Filebucket" do it "should create a remote Client to the configured server" do Puppet.stubs(:[]).with(:server).returns("puppet.reductivelabs.com") - Puppet::Network::Client::Dipper.expects(:new).with { |h| h[:Server] == "puppet.reductivelabs.com" } + Puppet::FileBucket::Dipper.expects(:new).with { |h| h[:Server] == "puppet.reductivelabs.com" } @filebucket.run_setup end @@ -142,11 +142,11 @@ describe "Filebucket" do Puppet.stubs(:settraps) Puppet::Log.stubs(:level=) Puppet.stubs(:parse_config) - Puppet::Network::Client.dipper.stubs(:new) + Puppet::FileBucket::Dipper.stubs(:new) @filebucket.options.stubs(:[]).with(any_parameters) @client = stub 'client' - Puppet::Network::Client::Dipper.stubs(:new).returns(@client) + Puppet::FileBucket::Dipper.stubs(:new).returns(@client) @filebucket.run_setup end diff --git a/spec/unit/application/main.rb b/spec/unit/application/main.rb index 74c40c14a..ea8c43fe6 100755 --- a/spec/unit/application/main.rb +++ b/spec/unit/application/main.rb @@ -53,7 +53,7 @@ describe "Puppet" do Puppet.stubs(:trap) Puppet::Log.stubs(:level=) Puppet.stubs(:parse_config) - Puppet::Network::Client.dipper.stubs(:new) + Puppet::FileBucket::Dipper.stubs(:new) STDIN.stubs(:read) @main.options.stubs(:[]).with(any_parameters) diff --git a/spec/unit/file_bucket/file.rb b/spec/unit/file_bucket/file.rb new file mode 100644 index 000000000..76f8e2599 --- /dev/null +++ b/spec/unit/file_bucket/file.rb @@ -0,0 +1,239 @@ +#!/usr/bin/env ruby + +require ::File.dirname(__FILE__) + '/../../spec_helper' + +require 'puppet/file_bucket/file' +require 'digest/md5' +require 'digest/sha1' + +describe Puppet::FileBucket::File do + before do + # this is the default from spec_helper, but it keeps getting reset at odd times + Puppet[:bucketdir] = "/dev/null/bucket" + + @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @checksum = "md5:4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @dir = '/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0' + + @contents = "file contents" + end + + it "should save a file" do + ::File.expects(:exist?).with("#{@dir}/contents").returns false + ::File.expects(:directory?).with(@dir).returns false + ::FileUtils.expects(:mkdir_p).with(@dir) + ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) + + bucketfile = Puppet::FileBucket::File.new(@contents) + bucketfile.save + + end + + describe "using the indirector's find method" do + it "should return nil if a file doesn't exist" do + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + bucketfile = Puppet::FileBucket::File.find("md5:#{@digest}") + bucketfile.should == nil + end + + it "should find a filebucket if the file exists" do + ::File.expects(:exist?).with("#{@dir}/contents").returns true + ::File.expects(:exist?).with("#{@dir}/paths").returns false + ::File.expects(:read).with("#{@dir}/contents").returns @contents + + bucketfile = Puppet::FileBucket::File.find("md5:#{@digest}") + bucketfile.should_not == nil + end + + describe "using RESTish digest notation" do + it "should return nil if a file doesn't exist" do + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + bucketfile = Puppet::FileBucket::File.find("md5/#{@digest}") + bucketfile.should == nil + end + + it "should find a filebucket if the file exists" do + ::File.expects(:exist?).with("#{@dir}/contents").returns true + ::File.expects(:exist?).with("#{@dir}/paths").returns false + ::File.expects(:read).with("#{@dir}/contents").returns @contents + + bucketfile = Puppet::FileBucket::File.find("md5/#{@digest}") + bucketfile.should_not == nil + end + + end + end + + it "should have a to_s method to return the contents" do + Puppet::FileBucket::File.new(@contents).to_s.should == @contents + end + + it "should have a method that returns the digest algorithm" do + Puppet::FileBucket::File.new(@contents, :checksum => @checksum).checksum_type.should == :md5 + end + + it "should allow contents to be specified in a block" do + bucket = Puppet::FileBucket::File.new(nil) do |fb| + fb.contents = "content" + end + bucket.contents.should == "content" + end + + it "should raise an error if changing content" do + x = Puppet::FileBucket::File.new("first") + proc { x.contents = "new" }.should raise_error + end + + it "should require contents to be a string" do + proc { Puppet::FileBucket::File.new(5) }.should raise_error(ArgumentError) + end + + it "should raise an error if setting contents to a non-string" do + proc do + Puppet::FileBucket::File.new(nil) do |x| + x.contents = 5 + end + end.should raise_error(ArgumentError) + end + + it "should set the contents appropriately" do + Puppet::FileBucket::File.new(@contents).contents.should == @contents + end + + it "should calculate the checksum" do + Digest::MD5.expects(:hexdigest).with(@contents).returns('mychecksum') + Puppet::FileBucket::File.new(@contents).checksum.should == 'md5:mychecksum' + end + + it "should remove the old checksum value if the algorithm is changed" do + Digest::MD5.expects(:hexdigest).with(@contents).returns('oldsum') + sum = Puppet::FileBucket::File.new(@contents) + oldsum = sum.checksum + + sum.checksum_type = :sha1 + Digest::SHA1.expects(:hexdigest).with(@contents).returns('newsum') + sum.checksum.should == 'sha1:newsum' + end + + it "should default to 'md5' as the checksum algorithm if the algorithm is not in the name" do + Puppet::FileBucket::File.new(@contents).checksum_type.should == :md5 + end + + it "should support specifying the checksum_type during initialization" do + sum = Puppet::FileBucket::File.new(@contents, :checksum_type => :sha1) + sum.checksum_type.should == :sha1 + end + + it "should fail when an unsupported checksum_type is used" do + proc { Puppet::FileBucket::File.new(@contents, :checksum_type => :nope) }.should raise_error(ArgumentError) + end + + it "should fail if given an invalid checksum at initialization" do + proc { Puppet::FileBucket::File.new(@contents, :checksum => "md5:00000000000000000000000000000000") }.should raise_error(RuntimeError) + end + + it "should fail if assigned an invalid checksum " do + bucket = Puppet::FileBucket::File.new(@contents) + proc { bucket.checksum = "md5:00000000000000000000000000000000" }.should raise_error(RuntimeError) + end + + it "should accept checksum_data without a prefix" do + bucket = Puppet::FileBucket::File.new(@contents) + bucket.checksum_data = @digest + end + + + describe "when using back-ends" do + it "should redirect using Puppet::Indirector" do + Puppet::Indirector::Indirection.instance(:file_bucket_file).model.should equal(Puppet::FileBucket::File) + end + + it "should have a :save instance method" do + Puppet::FileBucket::File.new("mysum").should respond_to(:save) + end + + it "should respond to :find" do + Puppet::FileBucket::File.should respond_to(:find) + end + + it "should respond to :destroy" do + Puppet::FileBucket::File.should respond_to(:destroy) + end + end + + describe "when saving files" do + it "should save the contents to the calculated path" do + ::File.stubs(:directory?).with(@dir).returns(true) + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + mockfile = mock "file" + mockfile.expects(:print).with(@contents) + ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440).yields(mockfile) + + Puppet::FileBucket::File.new(@contents).save + end + + it "should make any directories necessary for storage" do + FileUtils.expects(:mkdir_p).with do |arg| + ::File.umask == 0007 and arg == @dir + end + ::File.expects(:directory?).with(@dir).returns(false) + ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + Puppet::FileBucket::File.new(@contents).save + end + end + + it "should accept a path" do + remote_path = '/path/on/the/remote/box' + Puppet::FileBucket::File.new(@contents, :path => remote_path).path.should == remote_path + end + + it "should append the path to the paths file" do + remote_path = '/path/on/the/remote/box' + + ::File.expects(:directory?).with(@dir).returns(true) + ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + mockfile = mock "file" + mockfile.expects(:puts).with('/path/on/the/remote/box') + ::File.expects(:exist?).with("#{@dir}/paths").returns false + ::File.expects(:open).with("#{@dir}/paths", ::File::WRONLY|::File::CREAT|::File::APPEND).yields mockfile + Puppet::FileBucket::File.new(@contents, :path => remote_path).save + + end + + it "should return a url-ish name" do + Puppet::FileBucket::File.new(@contents).name.should == "md5/4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + end + + it "should reject a url-ish name with an invalid checksum" do + bucket = Puppet::FileBucket::File.new(@contents) + lambda { bucket.name = "sha1/4a8ec4fa5f01b4ab1a0ab8cbccb709f0/new/path" }.should raise_error + end + + it "should accept a url-ish name" do + bucket = Puppet::FileBucket::File.new(@contents) + lambda { bucket.name = "sha1/034fa2ed8e211e4d20f20e792d777f4a30af1a93/new/path" }.should_not raise_error + bucket.checksum_type.should == :sha1 + bucket.checksum_data.should == '034fa2ed8e211e4d20f20e792d777f4a30af1a93' + bucket.path.should == "new/path" + end + + it "should return a url-ish name with a path" do + Puppet::FileBucket::File.new(@contents, :path => 'my/path').name.should == "md5/4a8ec4fa5f01b4ab1a0ab8cbccb709f0/my/path" + end + + it "should convert the contents to PSON" do + Puppet::FileBucket::File.new(@contents).to_pson.should == '{"contents":"file contents"}' + end + + it "should load from PSON" do + Puppet::FileBucket::File.from_pson({"contents"=>"file contents"}).contents.should == "file contents" + end + +end diff --git a/spec/unit/indirector/file_bucket_file/file.rb b/spec/unit/indirector/file_bucket_file/file.rb new file mode 100755 index 000000000..0df530d74 --- /dev/null +++ b/spec/unit/indirector/file_bucket_file/file.rb @@ -0,0 +1,288 @@ +#!/usr/bin/env ruby + +require ::File.dirname(__FILE__) + '/../../../spec_helper' + +require 'puppet/indirector/file_bucket_file/file' + +describe Puppet::FileBucketFile::File do + it "should be a subclass of the Code terminus class" do + Puppet::FileBucketFile::File.superclass.should equal(Puppet::Indirector::Code) + end + + it "should have documentation" do + Puppet::FileBucketFile::File.doc.should be_instance_of(String) + end + + describe "when initializing" do + it "should use the filebucket settings section" do + Puppet.settings.expects(:use).with(:filebucket) + Puppet::FileBucketFile::File.new + end + end + + + describe "the find_by_checksum method" do + before do + # this is the default from spec_helper, but it keeps getting reset at odd times + Puppet[:bucketdir] = "/dev/null/bucket" + + @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @checksum = "md5:4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @dir = '/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0' + + @contents = "file contents" + end + + it "should return nil if a file doesn't exist" do + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + bucketfile = Puppet::FileBucketFile::File.new.send(:find_by_checksum, "md5:#{@digest}") + bucketfile.should == nil + end + + it "should find a filebucket if the file exists" do + ::File.expects(:exist?).with("#{@dir}/contents").returns true + ::File.expects(:exist?).with("#{@dir}/paths").returns false + ::File.expects(:read).with("#{@dir}/contents").returns @contents + + bucketfile = Puppet::FileBucketFile::File.new.send(:find_by_checksum, "md5:#{@digest}") + bucketfile.should_not == nil + end + + it "should load the paths" do + paths = ["path1", "path2"] + ::File.expects(:exist?).with("#{@dir}/contents").returns true + ::File.expects(:exist?).with("#{@dir}/paths").returns true + ::File.expects(:read).with("#{@dir}/contents").returns @contents + + mockfile = mock "file" + mockfile.expects(:readlines).returns( paths ) + ::File.expects(:open).with("#{@dir}/paths").yields mockfile + + Puppet::FileBucketFile::File.new.send(:find_by_checksum, "md5:#{@digest}").paths.should == paths + end + + end + + describe "when retrieving files" do + before :each do + Puppet.settings.stubs(:use) + @store = Puppet::FileBucketFile::File.new + + @digest = "70924d6fa4b2d745185fa4660703a5c0" + @sum = stub 'sum', :name => @digest + + @dir = "/what/ever" + + Puppet.stubs(:[]).with(:bucketdir).returns(@dir) + + @contents_path = '/what/ever/7/0/9/2/4/d/6/f/70924d6fa4b2d745185fa4660703a5c0/contents' + @paths_path = '/what/ever/7/0/9/2/4/d/6/f/70924d6fa4b2d745185fa4660703a5c0/paths' + + @request = stub 'request', :key => "md5/#{@digest}/remote/path" + end + + it "should call find_by_checksum" do + @store.expects(:find_by_checksum).with("md5:#{@digest}").returns(false) + @store.find(@request) + end + + it "should look for the calculated path" do + ::File.expects(:exist?).with(@contents_path).returns(false) + @store.find(@request) + end + + it "should return an instance of Puppet::FileBucket::File created with the content if the file exists" do + content = "my content" + bucketfile = stub 'bucketfile' + bucketfile.stubs(:bucket_path) + bucketfile.stubs(:checksum_data).returns(@digest) + bucketfile.stubs(:checksum).returns(@checksum) + + bucketfile.expects(:contents=).with(content) + Puppet::FileBucket::File.expects(:new).with(nil, {:checksum => "md5:#{@digest}"}).yields(bucketfile).returns(bucketfile) + + ::File.expects(:exist?).with(@contents_path).returns(true) + ::File.expects(:exist?).with(@paths_path).returns(false) + ::File.expects(:read).with(@contents_path).returns(content) + + @store.find(@request).should equal(bucketfile) + end + + it "should return nil if no file is found" do + ::File.expects(:exist?).with(@contents_path).returns(false) + @store.find(@request).should be_nil + end + + it "should fail intelligently if a found file cannot be read" do + ::File.expects(:exist?).with(@contents_path).returns(true) + ::File.expects(:read).with(@contents_path).raises(RuntimeError) + proc { @store.find(@request) }.should raise_error(Puppet::Error) + end + + end + + describe "when determining file paths" do + before do + Puppet[:bucketdir] = '/dev/null/bucketdir' + @digest = 'DEADBEEFC0FFEE' + @bucket = stub_everything "bucket" + @bucket.expects(:checksum_data).returns(@digest) + end + + it "should use the value of the :bucketdir setting as the root directory" do + path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) + path.should =~ %r{^/dev/null/bucketdir} + end + + it "should choose a path 8 directories deep with each directory name being the respective character in the filebucket" do + path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) + dirs = @digest[0..7].split("").join(File::SEPARATOR) + path.should be_include(dirs) + end + + it "should use the full filebucket as the final directory name" do + path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) + ::File.basename(::File.dirname(path)).should == @digest + end + + it "should use 'contents' as the actual file name" do + path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) + ::File.basename(path).should == "contents" + end + + it "should use the bucketdir, the 8 sum character directories, the full filebucket, and 'contents' as the full file name" do + path = Puppet::FileBucketFile::File.new.send(:contents_path_for, @bucket) + path.should == ['/dev/null/bucketdir', @digest[0..7].split(""), @digest, "contents"].flatten.join(::File::SEPARATOR) + end + end + + describe "when saving files" do + before do + # this is the default from spec_helper, but it keeps getting reset at odd times + Puppet[:bucketdir] = "/dev/null/bucket" + + @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @checksum = "md5:4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @dir = '/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0' + + @contents = "file contents" + + @bucket = stub "bucket file" + @bucket.stubs(:bucket_path) + @bucket.stubs(:checksum_data).returns(@digest) + @bucket.stubs(:path).returns(nil) + @bucket.stubs(:contents).returns("file contents") + end + + it "should save the contents to the calculated path" do + ::File.stubs(:directory?).with(@dir).returns(true) + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + mockfile = mock "file" + mockfile.expects(:print).with(@contents) + ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440).yields(mockfile) + + Puppet::FileBucketFile::File.new.send(:save_to_disk, @bucket) + end + + it "should make any directories necessary for storage" do + FileUtils.expects(:mkdir_p).with do |arg| + ::File.umask == 0007 and arg == @dir + end + ::File.expects(:directory?).with(@dir).returns(false) + ::File.expects(:open).with("#{@dir}/contents", ::File::WRONLY|::File::CREAT, 0440) + ::File.expects(:exist?).with("#{@dir}/contents").returns false + + Puppet::FileBucketFile::File.new.send(:save_to_disk, @bucket) + end + end + + + describe "when verifying identical files" do + before do + # this is the default from spec_helper, but it keeps getting reset at odd times + Puppet[:bucketdir] = "/dev/null/bucket" + + @digest = "4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @checksum = "md5:4a8ec4fa5f01b4ab1a0ab8cbccb709f0" + @dir = '/dev/null/bucket/4/a/8/e/c/4/f/a/4a8ec4fa5f01b4ab1a0ab8cbccb709f0' + + @contents = "file contents" + + @bucket = stub "bucket file" + @bucket.stubs(:bucket_path) + @bucket.stubs(:checksum).returns(@checksum) + @bucket.stubs(:checksum_data).returns(@digest) + @bucket.stubs(:path).returns(nil) + @bucket.stubs(:contents).returns("file contents") + end + + it "should raise an error if the files don't match" do + File.expects(:read).with("#{@dir}/contents").returns("corrupt contents") + lambda{ Puppet::FileBucketFile::File.new.send(:verify_identical_file!, @bucket) }.should raise_error(Puppet::FileBucket::BucketError) + end + + it "should do nothing if the files match" do + File.expects(:read).with("#{@dir}/contents").returns("file contents") + Puppet::FileBucketFile::File.new.send(:verify_identical_file!, @bucket) + end + + end + + + describe "when writing to the paths file" do + before do + Puppet[:bucketdir] = '/dev/null/bucketdir' + @digest = '70924d6fa4b2d745185fa4660703a5c0' + @bucket = stub_everything "bucket" + + @paths_path = '/dev/null/bucketdir/7/0/9/2/4/d/6/f/70924d6fa4b2d745185fa4660703a5c0/paths' + + @paths = [] + @bucket.stubs(:paths).returns(@paths) + @bucket.stubs(:checksum_data).returns(@digest) + end + + it "should create a file if it doesn't exist" do + @bucket.expects(:path).returns('path/to/save').at_least_once + File.expects(:exist?).with(@paths_path).returns(false) + file = stub "file" + file.expects(:puts).with('path/to/save') + File.expects(:open).with(@paths_path, ::File::WRONLY|::File::CREAT|::File::APPEND).yields(file) + + Puppet::FileBucketFile::File.new.send(:save_path_to_paths_file, @bucket) + end + + it "should append to a file if it exists" do + @bucket.expects(:path).returns('path/to/save').at_least_once + File.expects(:exist?).with(@paths_path).returns(true) + old_file = stub "file" + old_file.stubs(:readlines).returns [] + File.expects(:open).with(@paths_path).yields(old_file) + + file = stub "file" + file.expects(:puts).with('path/to/save') + File.expects(:open).with(@paths_path, ::File::WRONLY|::File::CREAT|::File::APPEND).yields(file) + + Puppet::FileBucketFile::File.new.send(:save_path_to_paths_file, @bucket) + end + + it "should not alter a file if it already contains the path" do + @bucket.expects(:path).returns('path/to/save').at_least_once + File.expects(:exist?).with(@paths_path).returns(true) + old_file = stub "file" + old_file.stubs(:readlines).returns ["path/to/save\n"] + File.expects(:open).with(@paths_path).yields(old_file) + + Puppet::FileBucketFile::File.new.send(:save_path_to_paths_file, @bucket) + end + + it "should do nothing if there is no path" do + @bucket.expects(:path).returns(nil).at_least_once + + Puppet::FileBucketFile::File.new.send(:save_path_to_paths_file, @bucket) + end + end + +end diff --git a/spec/unit/indirector/file_bucket_file/rest.rb b/spec/unit/indirector/file_bucket_file/rest.rb new file mode 100755 index 000000000..3aacd3ca4 --- /dev/null +++ b/spec/unit/indirector/file_bucket_file/rest.rb @@ -0,0 +1,11 @@ +#!/usr/bin/env ruby + +Dir.chdir(File.dirname(__FILE__)) { (s = lambda { |f| File.exist?(f) ? require(f) : Dir.chdir("..") { s.call(f) } }).call("spec/spec_helper.rb") } + +require 'puppet/indirector/file_bucket_file/rest' + +describe Puppet::FileBucketFile::Rest do + it "should be a sublcass of Puppet::Indirector::REST" do + Puppet::FileBucketFile::Rest.superclass.should equal(Puppet::Indirector::REST) + end +end diff --git a/spec/unit/indirector/indirection.rb b/spec/unit/indirector/indirection.rb index ca2a412e3..02d04a3f7 100755 --- a/spec/unit/indirector/indirection.rb +++ b/spec/unit/indirector/indirection.rb @@ -198,8 +198,8 @@ describe Puppet::Indirector::Indirection do @indirection.request(:funtest, "yayness", :one => :two) end - it "should default to the arguments being nil" do - Puppet::Indirector::Request.expects(:new).with { |name, method, key, args| args.nil? } + it "should default to the arguments being empty" do + Puppet::Indirector::Request.expects(:new).with { |name, method, key, args| args == {} } @indirection.request(:funtest, "yayness") end diff --git a/spec/unit/indirector/request.rb b/spec/unit/indirector/request.rb index 848a608a4..b885779ed 100755 --- a/spec/unit/indirector/request.rb +++ b/spec/unit/indirector/request.rb @@ -132,8 +132,8 @@ describe Puppet::Indirector::Request do Puppet::Indirector::Request.new(:ind, :method, "http://host/stuff").port.should == 80 end - it "should set the request key to the unescaped unqualified path from the URI" do - Puppet::Indirector::Request.new(:ind, :method, "http:///stuff with spaces").key.should == "stuff with spaces" + it "should set the request key to the unescaped key part path from the URI" do + Puppet::Indirector::Request.new(:ind, :method, "http://host/environment/terminus/stuff with spaces").key.should == "stuff with spaces" end it "should set the :uri attribute to the full URI" do diff --git a/spec/unit/network/client/dipper.rb b/spec/unit/network/client/dipper.rb index d1631fbb5..7d8b3da08 100755 --- a/spec/unit/network/client/dipper.rb +++ b/spec/unit/network/client/dipper.rb @@ -2,15 +2,109 @@ require File.dirname(__FILE__) + '/../../../spec_helper' -describe Puppet::Network::Client.dipper do +require 'puppet/file_bucket/dipper' +describe Puppet::FileBucket::Dipper do it "should fail in an informative way when there are failures backing up to the server" do - FileTest.stubs(:exists?).returns true + File.stubs(:exists?).returns true File.stubs(:read).returns "content" - @dipper = Puppet::Network::Client::Dipper.new(:Path => "/my/bucket") + @dipper = Puppet::FileBucket::Dipper.new(:Path => "/my/bucket") - @dipper.driver.expects(:addfile).raises ArgumentError + filemock = stub "bucketfile" + Puppet::FileBucket::File.stubs(:new).returns(filemock) + filemock.expects(:name).returns "name" + filemock.expects(:save).raises ArgumentError lambda { @dipper.backup("/my/file") }.should raise_error(Puppet::Error) end + + it "should backup files to a local bucket" do + @dipper = Puppet::FileBucket::Dipper.new( + :Path => "/my/bucket" + ) + + File.stubs(:exists?).returns true + File.stubs(:read).with("/my/file").returns "my contents" + + req = stub "req" + bucketfile = stub "bucketfile" + bucketfile.stubs(:name).returns('md5/DIGEST123') + bucketfile.stubs(:checksum_data).returns("DIGEST123") + bucketfile.expects(:save).with(req) + + Puppet::FileBucket::File.stubs(:new).with( + "my contents", + :bucket_path => '/my/bucket', + :path => '/my/file' + ).returns(bucketfile) + + Puppet::Indirector::Request.stubs(:new).with(:file_bucket_file, :save, 'md5/DIGEST123').returns(req) + + @dipper.backup("/my/file").should == "DIGEST123" + end + + it "should retrieve files from a local bucket" do + @dipper = Puppet::FileBucket::Dipper.new( + :Path => "/my/bucket" + ) + + File.stubs(:exists?).returns true + File.stubs(:read).with("/my/file").returns "my contents" + + bucketfile = stub "bucketfile" + bucketfile.stubs(:to_s).returns "Content" + + Puppet::FileBucket::File.expects(:find).with( + 'md5/DIGEST123' + ).returns(bucketfile) + + @dipper.getfile("DIGEST123").should == "Content" + end + + it "should backup files to a remote server" do + @dipper = Puppet::FileBucket::Dipper.new( + :Server => "puppetmaster", + :Port => "31337" + ) + + File.stubs(:exists?).returns true + File.stubs(:read).with("/my/file").returns "my contents" + + req = stub "req" + bucketfile = stub "bucketfile" + bucketfile.stubs(:name).returns('md5/DIGEST123') + bucketfile.stubs(:checksum_data).returns("DIGEST123") + bucketfile.expects(:save).with(req) + + Puppet::FileBucket::File.stubs(:new).with( + "my contents", + :bucket_path => nil, + :path => '/my/file' + ).returns(bucketfile) + + Puppet::Indirector::Request.stubs(:new).with(:file_bucket_file, :save, 'https://puppetmaster:31337/production/file_bucket_file/md5/DIGEST123').returns(req) + + @dipper.backup("/my/file").should == "DIGEST123" + end + + it "should retrieve files from a remote server" do + @dipper = Puppet::FileBucket::Dipper.new( + :Server => "puppetmaster", + :Port => "31337" + ) + + File.stubs(:exists?).returns true + File.stubs(:read).with("/my/file").returns "my contents" + + bucketfile = stub "bucketfile" + bucketfile.stubs(:to_s).returns "Content" + + Puppet::FileBucket::File.expects(:find).with( + 'https://puppetmaster:31337/production/file_bucket_file/md5/DIGEST123' + ).returns(bucketfile) + + @dipper.getfile("DIGEST123").should == "Content" + end + + end diff --git a/spec/unit/network/http/handler.rb b/spec/unit/network/http/handler.rb index 559812159..2218a0a78 100755 --- a/spec/unit/network/http/handler.rb +++ b/spec/unit/network/http/handler.rb @@ -402,10 +402,7 @@ describe Puppet::Network::HTTP::Handler do end it "should use a common method for determining the request parameters" do - @irequest.stubs(:to_hash).returns(:foo => :baz, :bar => :xyzzy) - @model_instance.expects(:save).with do |args| - args[:foo] == :baz and args[:bar] == :xyzzy - end + @model_instance.expects(:save).with(@irequest) @handler.do_save(@irequest, @request, @response) end diff --git a/spec/unit/other/checksum.rb b/spec/unit/other/checksum.rb deleted file mode 100755 index 6a63e833d..000000000 --- a/spec/unit/other/checksum.rb +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env ruby -# -# Created by Luke Kanies on 2007-9-22. -# Copyright (c) 2007. All rights reserved. - -require File.dirname(__FILE__) + '/../../spec_helper' - -require 'puppet/checksum' - -describe Puppet::Checksum do - it "should have 'Checksum' and the checksum algorithm when converted to a string" do - inst = Puppet::Checksum.new("whatever", "md5") - inst.to_s.should == "Checksum<{md5}#{inst.checksum}>" - end - - it "should convert algorithm names to symbols when they are set after checksum creation" do - sum = Puppet::Checksum.new("whatever") - sum.algorithm = "md5" - sum.algorithm.should == :md5 - end - - it "should return the checksum as the name" do - sum = Puppet::Checksum.new("whatever") - sum.checksum.should == sum.name - end -end - -describe Puppet::Checksum, " when initializing" do - before do - @content = "this is some content" - @sum = Puppet::Checksum.new(@content) - end - - it "should require content" do - proc { Puppet::Checksum.new(nil) }.should raise_error(ArgumentError) - end - - it "should set the content appropriately" do - @sum.content.should == @content - end - - it "should calculate the checksum" do - require 'digest/md5' - Digest::MD5.expects(:hexdigest).with(@content).returns(:mychecksum) - @sum.checksum.should == :mychecksum - end - - it "should not calculate the checksum until it is asked for" do - require 'digest/md5' - Digest::MD5.expects(:hexdigest).never - sum = Puppet::Checksum.new(@content, :md5) - end - - it "should remove the old checksum value if the algorithm is changed" do - Digest::MD5.expects(:hexdigest).with(@content).returns(:oldsum) - oldsum = @sum.checksum - @sum.algorithm = :sha1 - Digest::SHA1.expects(:hexdigest).with(@content).returns(:newsum) - @sum.checksum.should == :newsum - end - - it "should default to 'md5' as the checksum algorithm if the algorithm is not in the name" do - @sum.algorithm.should == :md5 - end - - it "should support specifying the algorithm during initialization" do - sum = Puppet::Checksum.new(@content, :sha1) - sum.algorithm.should == :sha1 - end - - it "should fail when an unsupported algorithm is used" do - proc { Puppet::Checksum.new(@content, :nope) }.should raise_error(ArgumentError) - end -end - -describe Puppet::Checksum, " when using back-ends" do - it "should redirect using Puppet::Indirector" do - Puppet::Indirector::Indirection.instance(:checksum).model.should equal(Puppet::Checksum) - end - - it "should have a :save instance method" do - Puppet::Checksum.new("mysum").should respond_to(:save) - end - - it "should respond to :find" do - Puppet::Checksum.should respond_to(:find) - end - - it "should respond to :destroy" do - Puppet::Checksum.should respond_to(:destroy) - end -end diff --git a/spec/unit/type/file.rb b/spec/unit/type/file.rb index afb050a1f..cedb1701d 100755 --- a/spec/unit/type/file.rb +++ b/spec/unit/type/file.rb @@ -811,7 +811,7 @@ describe Puppet::Type.type(:file) do it "should be able to use the default filebucket without a catalog" do file = Puppet::Type::File.new(:name => "/my/file", :backup => "puppet") - file.bucket.should be_instance_of(Puppet::Network::Client::Dipper) + file.bucket.should be_instance_of(Puppet::FileBucket::Dipper) end it "should look up the filebucket during finish()" do diff --git a/spec/unit/type/filebucket.rb b/spec/unit/type/filebucket.rb index 78bfa3655..a0ff45ab6 100644 --- a/spec/unit/type/filebucket.rb +++ b/spec/unit/type/filebucket.rb @@ -55,19 +55,19 @@ describe Puppet::Type.type(:filebucket) do it "should use any provided path" do bucket = Puppet::Type.type(:filebucket).new :name => "main", :path => "/foo/bar" - Puppet::Network::Client.client(:Dipper).expects(:new).with(:Path => "/foo/bar").returns @bucket + Puppet::FileBucket::Dipper.expects(:new).with(:Path => "/foo/bar").returns @bucket bucket.bucket end it "should use any provided server and port" do bucket = Puppet::Type.type(:filebucket).new :name => "main", :server => "myserv", :port => "myport", :path => false - Puppet::Network::Client.client(:Dipper).expects(:new).with(:Server => "myserv", :Port => "myport").returns @bucket + Puppet::FileBucket::Dipper.expects(:new).with(:Server => "myserv", :Port => "myport").returns @bucket bucket.bucket end it "should use the default server if the path is unset and no server is provided" do Puppet.settings[:server] = "myserv" bucket = Puppet::Type.type(:filebucket).new :name => "main", :path => false - Puppet::Network::Client.client(:Dipper).expects(:new).with { |args| args[:Server] == "myserv" }.returns @bucket + Puppet::FileBucket::Dipper.expects(:new).with { |args| args[:Server] == "myserv" }.returns @bucket bucket.bucket end end diff --git a/test/network/client/dipper.rb b/test/network/client/dipper.rb index 51494be9d..611ee7391 100755 --- a/test/network/client/dipper.rb +++ b/test/network/client/dipper.rb @@ -10,7 +10,7 @@ class TestDipperClient < Test::Unit::TestCase def setup super - @dipper = Puppet::Network::Client.dipper.new(:Path => tempfile) + @dipper = Puppet::FileBucket::Dipper.new(:Path => tempfile) end # Make sure we can create a new file with 'restore'. diff --git a/test/network/handler/bucket.rb b/test/network/handler/bucket.rb deleted file mode 100755 index 24e70cf01..000000000 --- a/test/network/handler/bucket.rb +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env ruby - -require File.dirname(__FILE__) + '/../../lib/puppettest' - -require 'puppettest' -require 'puppet/network/handler/filebucket' -require 'base64' -require 'mocha' - -class TestBucket < Test::Unit::TestCase - include PuppetTest::ServerTest - - def out - if defined? @num - @num += 1 - else - @num = 1 - end - - #Puppet.err "#{Process.pid}: %s: %s" % [@num, memory()] - #gcdebug(String) - end - - # run through all of the files and exercise the filebucket methods - def checkfiles(client) - files = filelist() - - # iterate across all of the files - files.each { |file| - Puppet.warning file - out - tempdir = tempfile() - Dir.mkdir(tempdir) - name = File.basename(file) - tmppath = File.join(tempdir,name) - @@tmpfiles << tmppath - - out - # copy the files to our tmp directory so we can modify them... - FileUtils.cp(file, tmppath) - - # make sure the copy worked - assert(FileTest.exists?(tmppath)) - - # backup both the orig file and the tmp file - osum = nil - tsum = nil - nsum = nil - out - assert_nothing_raised("Could not back up file") { - osum = client.backup(file) - } - out - assert_nothing_raised("Could not back up second file") { - tsum = client.backup(tmppath) - } - out - - # verify you got the same sum back for both - assert(tsum == osum) - - # modify our tmp file - unless FileTest.writable?(tmppath) - File.chmod(0644, tmppath) - end - File.open(tmppath,File::WRONLY|File::TRUNC) { |wf| - wf.print "This is some test text\n" - } - out - - # back it up - assert_nothing_raised { - #STDERR.puts("backing up %s" % tmppath) if $debug - nsum = client.backup(tmppath) - } - out - - # and verify the sum changed - assert(tsum != nsum) - - # restore the orig - assert_nothing_raised { - nsum = client.restore(tmppath,tsum) - } - out - - # and verify it actually got restored - contents = File.open(tmppath) { |rf| - #STDERR.puts("reading %s" % tmppath) if $debug - rf.read - } - out - csum = Digest::MD5.hexdigest(contents) - out - assert(tsum == csum) - } - end - - # a list of files that should be on the system - # just something to test moving files around - def filelist - if defined? @files - return @files - else - @files = [] - end - - %w{ - who bash sh uname /etc/passwd /etc/syslog.conf /etc/hosts - }.each { |file| - # if it's fully qualified, just add it - if file =~ /^\// - if FileTest.exists?(file) - @files.push file - end - else - # else if it's unqualified, look for it in our path - begin - path = %x{which #{file}} - rescue => detail - #STDERR.puts "Could not search for binaries: %s" % detail - next - end - - if path != "" - @files.push path.chomp - end - end - } - - return @files - end - - def setup - super - @bucket = tempfile() - end - - #def teardown - # system("lsof -p %s" % Process.pid) - # super - #end - - # test operating against the local filebucket object - # this calls the direct server methods, which are different than the - # Dipper methods - def test_localserver - files = filelist() - server = nil - assert_nothing_raised { - server = Puppet::Network::Handler.filebucket.new( - :Path => @bucket - ) - } - - # iterate across them... - files.each { |file| - contents = File.open(file) { |of| of.read } - - md5 = nil - - # add a file to the repository - assert_nothing_raised { - #STDERR.puts("adding %s" % file) if $debug - md5 = server.addfile(Base64.encode64(contents),file) - } - - # and get it back again - newcontents = nil - assert_nothing_raised { - #STDERR.puts("getting %s" % file) if $debug - newcontents = Base64.decode64(server.getfile(md5)) - } - - # and then make sure they're still the same - assert( - contents == newcontents - ) - } - end - - # test with a server and a Dipper - def test_localboth - files = filelist() - - bucket = nil - client = nil - threads = [] - assert_nothing_raised { - bucket = Puppet::Network::Handler.filebucket.new( - :Path => @bucket - ) - } - - #sleep(30) - assert_nothing_raised { - client = Puppet::Network::Client.dipper.new( - :Bucket => bucket - ) - } - - #4.times { checkfiles(client) } - checkfiles(client) - end - - def test_no_path_duplicates - bucket = nil - assert_nothing_raised { - bucket = Puppet::Network::Handler.filebucket.new( - :Path => @bucket - ) - } - - sum = nil - assert_nothing_raised { - sum = bucket.addfile("yayness", "/my/file") - } - assert_nothing_raised { - bucket.addfile("yayness", "/my/file") - } - - a, b, pathfile = bucket.class.paths(bucket.path, sum) - - assert(FileTest.exists?(pathfile), "No path file at %s" % pathfile) - - assert_equal("/my/file\n", File.read(pathfile)) - end - - # #447 -- a flat file structure just won't suffice. - def test_deeper_filestructure - bucket = Puppet::Network::Handler.filebucket.new(:Path => @bucket) - - text = "this is some text" - md5 = Digest::MD5.hexdigest(text) - - olddir = File.join(@bucket, md5) - FileUtils.mkdir_p(olddir) - oldcontent = File.join(olddir, "contents") - File.open(oldcontent, "w") { |f| f.print text } - - result = nil - assert_nothing_raised("Could not retrieve content from old structure") do - result = bucket.getfile(md5) - end - assert_equal(text, result, "old-style content is wrong") - - text = "and this is some new text" - md5 = Digest::MD5.hexdigest(text) - - dirs = File.join(md5[0..7].split("")) - dir = File.join(@bucket, dirs, md5) - filedir, contents, paths = bucket.class.paths(@bucket, md5) - - assert_equal(dir, filedir, "did not use a deeper file structure") - assert_equal(File.join(dir, "contents"), contents, - "content path is not the deeper version") - assert_equal(File.join(dir, "paths"), paths, - "paths file path is not the deeper version") - - # Store our new text and make sure it gets stored in the new location - path = "/some/fake/path" - assert_nothing_raised("Could not store text") do - bucket.addfile(text, path) - end - assert(FileTest.exists?(contents), "did not create content file") - assert_equal(text, File.read(contents), "content is not right") - assert(FileTest.exists?(paths), "did not create paths file") - assert(File.read(paths).include?(path), "paths file does not contain path") - - # And make sure we get it back out again - assert_nothing_raised("Could not retrieve new-style content") do - result = bucket.getfile(md5) - end - assert_equal(text, result, "did not retrieve new content correctly") - end - - def test_add_path - bucket = Puppet::Network::Handler.filebucket.new(:Path => @bucket) - - file = tempfile() - - assert(! FileTest.exists?(file), "file already exists") - - path = "/some/thing" - assert_nothing_raised("Could not add path") do - bucket.send(:add_path, path, file) - end - assert_equal(path + "\n", File.read(file), "path was not added") - - assert_nothing_raised("Could not add path second time") do - bucket.send(:add_path, path, file) - end - assert_equal(path + "\n", File.read(file), "path was duplicated") - - # Now try a new path - newpath = "/another/path" - assert_nothing_raised("Could not add path second time") do - bucket.send(:add_path, newpath, file) - end - text = [path, newpath].join("\n") + "\n" - assert_equal(text, File.read(file), "path was duplicated") - - assert_nothing_raised("Could not add path third time") do - bucket.send(:add_path, path, file) - end - assert_equal(text, File.read(file), "path was duplicated") - assert_nothing_raised("Could not add second path second time") do - bucket.send(:add_path, newpath, file) - end - assert_equal(text, File.read(file), "path was duplicated") - end -end - |