diff options
| -rw-r--r-- | lib/puppet/parser/collector.rb | 164 | ||||
| -rwxr-xr-x | spec/unit/parser/collector.rb | 386 | ||||
| -rwxr-xr-x | test/language/collector.rb | 178 |
3 files changed, 461 insertions, 267 deletions
diff --git a/lib/puppet/parser/collector.rb b/lib/puppet/parser/collector.rb index e3c91bccb..6dcded2f9 100644 --- a/lib/puppet/parser/collector.rb +++ b/lib/puppet/parser/collector.rb @@ -1,7 +1,63 @@ # An object that collects stored objects from the central cache and returns # them to the current host, yo. class Puppet::Parser::Collector - attr_accessor :type, :scope, :vquery, :rquery, :form, :resources + attr_accessor :type, :scope, :vquery, :equery, :form, :resources + + # Call the collection method, mark all of the returned objects as non-virtual, + # and then delete this object from the list of collections to evaluate. + def evaluate + if self.resources + if objects = collect_resources and ! objects.empty? + return objects + else + return false + end + else + method = "collect_#{@form.to_s}" + changer = @form.to_s + "=" + objects = send(method).each { |obj| obj.send(changer, false) } + if objects.empty? + return false + else + return objects + end + end + end + + def initialize(scope, type, equery, vquery, form) + @scope = scope + @type = type + @equery = equery + @vquery = vquery + + raise(ArgumentError, "Invalid query form %s" % form) unless [:exported, :virtual].include?(form) + @form = form + end + + private + + # Create our active record query. + def build_active_record_query + Puppet::Rails.init unless ActiveRecord::Base.connected? + + raise Puppet::DevError, "Cannot collect resources for a nil host" unless @scope.host + host = Puppet::Rails::Host.find_by_name(@scope.host) + + query = {:include => {:param_values => :param_name}} + + search = "(exported=? AND restype=?)" + values = [true, @type] + + search += " AND (?)" and values << @equery if @equery + + # We're going to collect objects from rails, but we don't want any + # objects from this host. + search = ("host_id != ? AND %s" % search) and values.unshift(host.id) if host + + query[:conditions] = [search, *values] + + return query + end # Collect exported objects. def collect_exported @@ -9,39 +65,16 @@ class Puppet::Parser::Collector # collect_virtual method but tell it to use 'exported? for the test. resources = collect_virtual(true).reject { |r| ! r.virtual? } - count = 0 - - unless @scope.host - raise Puppet::DevError, "Cannot collect resources for a nil host" - end - - # We're going to collect objects from rails, but we don't want any - # objects from this host. - unless ActiveRecord::Base.connected? - Puppet::Rails.init - end - host = Puppet::Rails::Host.find_by_name(@scope.host) + count = resources.length - args = {:include => {:param_values => :param_name}} - args[:conditions] = "(exported = %s AND restype = '%s')" % - [ActiveRecord::Base.connection.quote(true), @type] - if @equery - args[:conditions] += " AND (%s)" % [@equery] - end - if host - args[:conditions] = "host_id != %s AND %s" % [host.id, args[:conditions]] - else - #Puppet.info "Host %s is uninitialized" % @scope.host - end + query = build_active_record_query # Now look them up in the rails db. When we support attribute comparison # and such, we'll need to vary the conditions, but this works with no # attributes, anyway. time = Puppet::Util.thinmark do - Puppet::Rails::Resource.find(:all, @type, true, - args - ).each do |obj| - if resource = export_resource(obj) + Puppet::Rails::Resource.find(:all, @type, true, query).each do |obj| + if resource = exported_resource(obj) count += 1 resources << resource end @@ -70,6 +103,7 @@ class Puppet::Parser::Collector # Collect resources directly; this is the result of using 'realize', # which specifies resources, rather than using a normal collection. def collect_virtual_resources + return [] unless defined?(@resources) and ! @resources.empty? result = @resources.dup.collect do |ref| if res = @scope.findresource(ref.to_s) @resources.delete(ref) @@ -100,33 +134,21 @@ class Puppet::Parser::Collector end end - # Call the collection method, mark all of the returned objects as non-virtual, - # and then delete this object from the list of collections to evaluate. - def evaluate - if self.resources - if objects = collect_resources and ! objects.empty? - return objects - else - return false - end - else - method = "collect_#{@form.to_s}" - objects = send(method).each { |obj| obj.virtual = false } - if objects.empty? - return false - else - return objects - end + # Seek a specific exported resource. + def exported_resource(obj) + if existing = @scope.findresource(obj.restype, obj.title) + # Next see if we've already collected this resource + return nil if existing.rails_id == obj.id + + # This is the one we've already collected + raise Puppet::ParseError, "Exported resource %s cannot override local resource" % [obj.ref] end - end - def initialize(scope, type, equery, vquery, form) - @scope = scope - @type = type - @equery = equery - @vquery = vquery - @form = form - @tests = [] + resource = obj.to_resource(self.scope) + + scope.compile.store_resource(scope, resource) + + return resource end # Does the resource match our tests? We don't yet support tests, @@ -138,40 +160,4 @@ class Puppet::Parser::Collector return true end end - - def export_resource(obj) - if existing = @scope.findresource(obj.restype, obj.title) - # See if we exported it; if so, just move on - if @scope.host == obj.host.name - return nil - else - # Next see if we've already collected this resource - if existing.rails_id == obj.id - # This is the one we've already collected - return nil - else - raise Puppet::ParseError, - "Exported resource %s cannot override local resource" % - [obj.ref] - end - end - end - - begin - resource = obj.to_resource(self.scope) - - # XXX Because the scopes don't expect objects to return values, - # we have to manually add our objects to the scope. This is - # ber-lame. - scope.compile.store_resource(scope, resource) - rescue => detail - if Puppet[:trace] - puts detail.backtrace - end - raise - end - resource.exported = false - - return resource - end end diff --git a/spec/unit/parser/collector.rb b/spec/unit/parser/collector.rb new file mode 100755 index 000000000..caeb2b5a3 --- /dev/null +++ b/spec/unit/parser/collector.rb @@ -0,0 +1,386 @@ +#!/usr/bin/env ruby + +require File.dirname(__FILE__) + '/../../spec_helper' + +require 'puppet/parser/collector' + +describe Puppet::Parser::Collector, "when initializing" do + before do + @scope = mock 'scope' + @resource_type = mock 'resource_type' + @form = :exported + @vquery = mock 'vquery' + @equery = mock 'equery' + + @collector = Puppet::Parser::Collector.new(@scope, @resource_type, @equery, @vquery, @form) + end + + it "should require a scope" do + @collector.scope.should equal(@scope) + end + + it "should require a resource type" do + @collector.type.should equal(@resource_type) + end + + it "should only accept :virtual or :exported as the collector form" do + proc { @collector = Puppet::Parser::Collector.new(@scope, @resource_type, @vquery, @equery, :other) }.should raise_error(ArgumentError) + end + + it "should accept an optional virtual query" do + @collector.vquery.should equal(@vquery) + end + + it "should accept an optional exported query" do + @collector.equery.should equal(@equery) + end +end + +describe Puppet::Parser::Collector, "when collecting specific virtual resources" do + before do + @scope = mock 'scope' + @resource_type = mock 'resource_type' + @vquery = mock 'vquery' + @equery = mock 'equery' + + @collector = Puppet::Parser::Collector.new(@scope, @resource_type, @equery, @vquery, :virtual) + end + + it "should not fail when it does not find any resources to collect" do + @collector.resources = ["File[virtual1]", "File[virtual2]"] + @scope.stubs(:findresource).returns(false) + proc { @collector.evaluate }.should_not raise_error + end + + it "should mark matched resources as non-virtual" do + @collector.resources = ["File[virtual1]", "File[virtual2]"] + one = mock 'one' + one.expects(:virtual=).with(false) + @scope.stubs(:findresource).with("File[virtual1]").returns(one) + @scope.stubs(:findresource).with("File[virtual2]").returns(nil) + @collector.evaluate + end + + it "should return matched resources" do + @collector.resources = ["File[virtual1]", "File[virtual2]"] + one = mock 'one' + one.stubs(:virtual=) + @scope.stubs(:findresource).with("File[virtual1]").returns(one) + @scope.stubs(:findresource).with("File[virtual2]").returns(nil) + @collector.evaluate.should == [one] + end + + it "should delete itself from the compile's collection list if it has found all of its resources" do + @collector.resources = ["File[virtual1]"] + one = mock 'one' + one.stubs(:virtual=) + @compile.expects(:delete_collection).with(@collector) + @scope.expects(:compile).returns(@compile) + @scope.stubs(:findresource).with("File[virtual1]").returns(one) + @collector.evaluate + end + + it "should not delete itself from the compile's collection list if it has unfound resources" do + @collector.resources = ["File[virtual1]"] + one = mock 'one' + one.stubs(:virtual=) + @compile.expects(:delete_collection).never + @scope.stubs(:findresource).with("File[virtual1]").returns(nil) + @collector.evaluate + end +end + +describe Puppet::Parser::Collector, "when collecting virtual resources" do + before do + @scope = mock 'scope' + @compile = mock 'compile' + @scope.stubs(:compile).returns(@compile) + @resource_type = :mytype + @vquery = proc { |res| true } + + @collector = Puppet::Parser::Collector.new(@scope, @resource_type, nil, @vquery, :virtual) + end + + it "should find all resources matching the vquery" do + one = stub 'one', :type => :mytype, :virtual? => true + two = stub 'two', :type => :mytype, :virtual? => true + + one.stubs(:virtual=) + two.stubs(:virtual=) + + @compile.expects(:resources).returns([one, two]) + + @collector.evaluate.should == [one, two] + end + + it "should mark all matched resources as non-virtual" do + one = stub 'one', :type => :mytype, :virtual? => true + + one.expects(:virtual=).with(false) + + @compile.expects(:resources).returns([one]) + + @collector.evaluate + end + + it "should return matched resources" do + one = stub 'one', :type => :mytype, :virtual? => true + two = stub 'two', :type => :mytype, :virtual? => true + + one.stubs(:virtual=) + two.stubs(:virtual=) + + @compile.expects(:resources).returns([one, two]) + + @collector.evaluate.should == [one, two] + end + + it "should return all resources of the correct type if there is no virtual query" do + one = stub 'one', :type => :mytype, :virtual? => true + two = stub 'two', :type => :mytype, :virtual? => true + + one.expects(:virtual=).with(false) + two.expects(:virtual=).with(false) + + @compile.expects(:resources).returns([one, two]) + + @collector = Puppet::Parser::Collector.new(@scope, @resource_type, nil, nil, :virtual) + + @collector.evaluate.should == [one, two] + end + + it "should not return or mark resources of a different type" do + one = stub 'one', :type => :mytype, :virtual? => true + two = stub 'two', :type => :other, :virtual? => true + + one.expects(:virtual=).with(false) + two.expects(:virtual=).never + + @compile.expects(:resources).returns([one, two]) + + @collector.evaluate.should == [one] + end + + it "should not return or mark non-virtual resources" do + one = stub 'one', :type => :mytype, :virtual? => false + two = stub 'two', :type => :other, :virtual? => false + + one.expects(:virtual=).never + two.expects(:virtual=).never + + @compile.expects(:resources).returns([one, two]) + + @collector.evaluate.should be_false + end + + it "should not return or mark non-matching resources" do + @collector.vquery = proc { |res| res.name == :one } + + one = stub 'one', :name => :one, :type => :mytype, :virtual? => true + two = stub 'two', :name => :two, :type => :mytype, :virtual? => true + + one.expects(:virtual=).with(false) + two.expects(:virtual=).never + + @compile.expects(:resources).returns([one, two]) + + @collector.evaluate.should == [one] + end +end + +describe Puppet::Parser::Collector, "when collecting exported resources" do + confine Puppet.features.rails? => "Cannot test Rails integration without ActiveRecord" + + before do + @scope = stub 'scope', :host => "myhost", :debug => nil + @compile = mock 'compile' + @scope.stubs(:compile).returns(@compile) + @resource_type = :mytype + @equery = "test = true" + @vquery = proc { |r| true } + + @collector = Puppet::Parser::Collector.new(@scope, @resource_type, @equery, @vquery, :exported) + end + + # Stub most of our interface to Rails. + def stub_rails(everything = false) + ActiveRecord::Base.stubs(:connected?).returns(false) + Puppet::Rails.stubs(:init) + if everything + Puppet::Rails::Host.stubs(:find_by_name).returns(nil) + Puppet::Rails::Resource.stubs(:find).returns([]) + end + end + + it "should use initialize the Rails support if ActiveRecord is not connected" do + @compile.stubs(:resources).returns([]) + ActiveRecord::Base.expects(:connected?).returns(false) + Puppet::Rails.expects(:init) + Puppet::Rails::Host.stubs(:find_by_name).returns(nil) + Puppet::Rails::Resource.stubs(:find).returns([]) + + @collector.evaluate + end + + it "should return all matching resources from the current compile" do + stub_rails(true) + + one = stub 'one', :type => :mytype, :virtual? => true, :exported? => true + two = stub 'two', :type => :mytype, :virtual? => true, :exported? => true + + one.expects(:exported=).with(false) + two.expects(:exported=).with(false) + + @compile.expects(:resources).returns([one, two]) + + @collector.evaluate.should == [one, two] + end + + it "should mark all returned resources as not exported" do + stub_rails(true) + + one = stub 'one', :type => :mytype, :virtual? => true, :exported? => true + + one.expects(:exported=).with(false) + + @compile.expects(:resources).returns([one]) + + @collector.evaluate.should == [one] + end + + it "should convert all found resources into parser resources" do + stub_rails() + Puppet::Rails::Host.stubs(:find_by_name).returns(nil) + + one = stub 'one', :restype => :mytype, :title => "one", :virtual? => true, :exported? => true + Puppet::Rails::Resource.stubs(:find).returns([one]) + + resource = mock 'resource' + one.expects(:to_resource).with(@scope).returns(resource) + resource.expects(:exported=).with(false) + + @compile.stubs(:resources).returns([]) + @scope.stubs(:findresource).returns(nil) + + @compile.stubs(:store_resource) + + @collector.evaluate.should == [resource] + end + + it "should store converted resources in the compile's resource list" do + stub_rails() + Puppet::Rails::Host.stubs(:find_by_name).returns(nil) + + one = stub 'one', :restype => :mytype, :title => "one", :virtual? => true, :exported? => true + Puppet::Rails::Resource.stubs(:find).returns([one]) + + resource = mock 'resource' + one.expects(:to_resource).with(@scope).returns(resource) + resource.expects(:exported=).with(false) + + @compile.stubs(:resources).returns([]) + @scope.stubs(:findresource).returns(nil) + + @compile.expects(:store_resource).with(@scope, resource) + + @collector.evaluate.should == [resource] + end + + it "should fail if an equivalent resource already exists in the compile" do + stub_rails() + Puppet::Rails::Host.stubs(:find_by_name).returns(nil) + + rails = stub 'one', :restype => :mytype, :title => "one", :virtual? => true, :exported? => true, :id => 1, :ref => "yay" + inmemory = stub 'one', :type => :mytype, :virtual? => true, :exported? => true, :rails_id => 2 + + Puppet::Rails::Resource.stubs(:find).returns([rails]) + + resource = mock 'resource' + + @compile.stubs(:resources).returns([]) + @scope.stubs(:findresource).returns(inmemory) + + @compile.stubs(:store_resource) + + proc { @collector.evaluate }.should raise_error(Puppet::ParseError) + end + + it "should ignore exported resources that match already-collected resources" do + stub_rails() + Puppet::Rails::Host.stubs(:find_by_name).returns(nil) + + rails = stub 'one', :restype => :mytype, :title => "one", :virtual? => true, :exported? => true, :id => 1, :ref => "yay" + inmemory = stub 'one', :type => :mytype, :virtual? => true, :exported? => true, :rails_id => 1 + + Puppet::Rails::Resource.stubs(:find).returns([rails]) + + resource = mock 'resource' + + @compile.stubs(:resources).returns([]) + @scope.stubs(:findresource).returns(inmemory) + + @compile.stubs(:store_resource) + + proc { @collector.evaluate }.should_not raise_error(Puppet::ParseError) + end +end + +describe Puppet::Parser::Collector, "when building its ActiveRecord query for collecting exported resources" do + confine Puppet.features.rails? => "Cannot test Rails integration without ActiveRecord" + + before do + @scope = stub 'scope', :host => "myhost", :debug => nil + @compile = mock 'compile' + @scope.stubs(:compile).returns(@compile) + @resource_type = :mytype + @equery = nil + @vquery = proc { |r| true } + + @collector = Puppet::Parser::Collector.new(@scope, @resource_type, @equery, @vquery, :exported) + @compile.stubs(:resources).returns([]) + + ActiveRecord::Base.stubs(:connected?).returns(false) + + Puppet::Rails.stubs(:init) + Puppet::Rails::Host.stubs(:find_by_name).returns(nil) + Puppet::Rails::Resource.stubs(:find).returns([]) + end + + it "should exclude all resources from the host if ActiveRecord contains information for this host" do + @host = mock 'host' + @host.stubs(:id).returns 5 + + Puppet::Rails::Host.expects(:find_by_name).with(@scope.host).returns(@host) + + Puppet::Rails::Resource.stubs(:find).with { |*arguments| + options = arguments[3] + options[:conditions][0] =~ /^host_id != \?/ and options[:conditions][1] == 5 + }.returns([]) + + @collector.evaluate + end + + it "should return parameter names and parameter values when querying ActiveRecord" do + Puppet::Rails::Resource.stubs(:find).with { |*arguments| + options = arguments[3] + options[:include] == {:param_values => :param_name} + }.returns([]) + + @collector.evaluate + end + + it "should only search for exported resources with the matching type" do + Puppet::Rails::Resource.stubs(:find).with { |*arguments| + options = arguments[3] + options[:conditions][0].include?("(exported=? AND restype=?)") and options[:conditions][1] == true and options[:conditions][2] == :mytype + }.returns([]) + end + + it "should include the export query if one is provided" do + @collector = Puppet::Parser::Collector.new(@scope, @resource_type, "test = true", @vquery, :exported) + Puppet::Rails::Resource.stubs(:find).with { |*arguments| + options = arguments[3] + options[:conditions][0].include?("test = true") + }.returns([]) + end +end diff --git a/test/language/collector.rb b/test/language/collector.rb deleted file mode 100755 index 30949c061..000000000 --- a/test/language/collector.rb +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env ruby - -require File.dirname(__FILE__) + '/../lib/puppettest' - -require 'puppettest' -require 'puppettest/parsertesting' -require 'puppettest/resourcetesting' - -class TestCollector < Test::Unit::TestCase - include PuppetTest - include PuppetTest::ParserTesting - include PuppetTest::ResourceTesting - Parser = Puppet::Parser - AST = Parser::AST - - def setup - super - Puppet[:trace] = false - @scope = mkscope - @compile = @scope.compile - end - - # Test just collecting a specific resource. This is used by the 'realize' - # function, and it's much faster than iterating over all of the resources. - def test_collect_resource - # Make a collector - coll = nil - assert_nothing_raised do - coll = Puppet::Parser::Collector.new(@scope, "file", nil, nil, :virtual) - end - - # Now set the resource in the collector - assert_nothing_raised do - coll.resources = ["File[/tmp/virtual1]", "File[/tmp/virtual3]"] - end - @compile.add_collection(coll) - - # Evaluate the collector and make sure it doesn't fail with no resources - # found yet - assert_nothing_raised("Resource collection with no results failed") do - assert_equal(false, coll.evaluate) - end - - # Make a couple of virtual resources - one = mkresource(:type => "file", :title => "/tmp/virtual1", - :virtual => true, :params => {:owner => "root"}) - two = mkresource(:type => "file", :title => "/tmp/virtual2", - :virtual => true, :params => {:owner => "root"}) - @scope.compile.store_resource @scope, one - @scope.compile.store_resource @scope, two - - # Now run the collector again and make sure it finds our resource - assert_nothing_raised do - assert_equal([one], coll.evaluate, "did not find resource") - end - - # And make sure the resource is no longer virtual - assert(! one.virtual?, - "Resource is still virtual") - - # But the other still is - assert(two.virtual?, - "Resource got realized") - - # Make sure that the collection is still there - assert(@compile.collections.include?(coll), "collection was deleted too soon") - - # Now add our third resource - three = mkresource(:type => "file", :title => "/tmp/virtual3", - :virtual => true, :params => {:owner => "root"}) - @scope.compile.store_resource @scope, three - - # Run the collection - assert_nothing_raised do - assert_equal([three], coll.evaluate, "did not find resource") - end - assert(! three.virtual?, "three is still virtual") - - # And make sure that the collection got deleted from the scope's list - assert(@compile.collections.empty?, "collection was not deleted") - end - - def test_virtual - # Make a virtual resource - virtual = mkresource(:type => "file", :title => "/tmp/virtual", - :virtual => true, :params => {:owner => "root"}) - @scope.compile.store_resource @scope, virtual - - # And a non-virtual - real = mkresource(:type => "file", :title => "/tmp/real", - :params => {:owner => "root"}) - @scope.compile.store_resource @scope, real - - # Now make a collector - coll = nil - - # Make a fake query - code = proc do |res| - true - end - assert_nothing_raised do - coll = Puppet::Parser::Collector.new(@scope, "file", nil, code, :virtual) - end - - # Set it in our scope - @compile.add_collection(coll) - - # Make sure it's in the collections - assert(@compile.collections.include?(coll), "collection was not added") - - # And try to collect the virtual resources. - ret = nil - assert_nothing_raised do - ret = coll.collect_virtual - end - - assert_equal([virtual], ret) - - # Now make sure evaluate does the right thing. - assert_nothing_raised do - ret = coll.evaluate - end - - # And make sure our virtual object is no longer virtual - assert(! virtual.virtual?, "Virtual object did not get realized") - - # Now make a new collector of a different type and make sure it - # finds nothing. - assert_nothing_raised do - coll = Puppet::Parser::Collector.new(@scope, "exec", nil, nil, :virtual) - end - - # Remark this as virtual - virtual.virtual = true - - assert_nothing_raised do - ret = coll.evaluate - end - - assert_equal(false, ret) - end - - # Collections that specify resources should be deleted when they succeed, - # but others should remain until the very end. - def test_normal_collections_remain - # Make a collector - coll = nil - assert_nothing_raised do - coll = Puppet::Parser::Collector.new(@scope, "file", nil, nil, :virtual) - end - - @compile.add_collection(coll) - - # run the collection and make sure it doesn't get deleted, since it - # didn't return anything - assert_nothing_raised do - assert_equal(false, coll.evaluate, - "Evaluate returned incorrect value") - end - - assert_equal([coll], @compile.collections, "Collection was deleted") - - # Make a resource - one = mkresource(:type => "file", :title => "/tmp/virtual1", - :virtual => true, :params => {:owner => "root"}) - @scope.compile.store_resource @scope, one - - # Now perform the collection again, and it should still be there - assert_nothing_raised do - assert_equal([one], coll.evaluate, - "Evaluate returned incorrect value") - end - - assert_equal([coll], @compile.collections, "Collection was deleted") - - assert_equal(false, one.virtual?, "One was not realized") - end -end |