Feature #2378 - Implement "thin_storeconfigs"

Thin storeconfigs is a limited version of storeconfigs that is
more performant and still allows the exported/collected resources
system wich is the primary use of storeconfigs.

It works by storing to the database only the exported resources, tags
and host facts.

Since usually those exported resources are less than the number
of total resources for a node, it is expected to be faster than
regular storeconfigs (especially for the first run).

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

4 files changed, 100 insertions, 2 deletions

diff --git a/lib/puppet/defaults.rb b/lib/puppet/defaults.rb
index b2de823a5..4fb8bfeba 100644
--- a/lib/puppet/defaults.rb
+++ b/lib/puppet/defaults.rb
@@ -180,7 +180,15 @@ module Puppet
                     raise "Cannot disable asynchronous storeconfigs in a running process"
                 end
             end
-        }
+        },
+        :thin_storeconfigs => {:default => false, :desc =>
+            "Boolean; wether storeconfigs store in the database only the facts and exported resources.
+            If true, then storeconfigs performance will be higher and still allow exported/collected
+            resources, but other usage external to Puppet might not work",
+            :hook => proc do |value|
+                    Puppet.settings[:storeconfigs] = true if value
+                end
+            }
     )
 
     hostname = Facter["hostname"].value
diff --git a/lib/puppet/rails/host.rb b/lib/puppet/rails/host.rb
index 14ad5e6aa..d66fd2ed7 100644
--- a/lib/puppet/rails/host.rb
+++ b/lib/puppet/rails/host.rb
@@ -145,6 +145,9 @@ class Puppet::Rails::Host < ActiveRecord::Base
 
     # Set our resources.
     def merge_resources(list)
+        # keep only exported resources in thin_storeconfig mode
+        list = list.select { |r| r.exported? } if Puppet.settings[:thin_storeconfigs]
+
         resources_by_id = nil
         debug_benchmark("Searched for resources") {
             resources_by_id = find_resources()
@@ -160,7 +163,9 @@ class Puppet::Rails::Host < ActiveRecord::Base
     end
 
     def find_resources
-        resources.find(:all, :include => :source_file).inject({}) do | hash, resource |
+        condition = { :exported => true } if Puppet.settings[:thin_storeconfigs]
+
+        resources.find(:all, :include => :source_file, :conditions => condition || {}).inject({}) do | hash, resource |
             hash[resource.id] = resource
             hash
         end
diff --git a/spec/integration/defaults.rb b/spec/integration/defaults.rb
index a99a54435..e52eb5352 100755
--- a/spec/integration/defaults.rb
+++ b/spec/integration/defaults.rb
@@ -140,4 +140,17 @@ describe "Puppet defaults" do
             Puppet.settings[:storeconfigs] = true
         end
     end
+
+    describe "when enabling thing storeconfigs" do
+        before do
+            Puppet::Resource::Catalog.stubs(:cache_class=)
+            Puppet::Node::Facts.stubs(:cache_class=)
+            Puppet::Node.stubs(:cache_class=)
+        end
+
+        it "should set storeconfigs to true" do
+            Puppet.settings[:thin_storeconfigs] = true
+            Puppet.settings[:storeconfigs].should be_true
+        end
+    end
 end
diff --git a/spec/unit/rails/host.rb b/spec/unit/rails/host.rb
index 882abbd5a..45d4808fe 100755
--- a/spec/unit/rails/host.rb
+++ b/spec/unit/rails/host.rb
@@ -88,4 +88,76 @@ describe "Puppet::Rails::Host" do
             @host.to_puppet
         end
     end
+
+    describe "when merging catalog resources and database resources" do
+        before :each do
+            Puppet.settings.stubs(:[]).with(:thin_storeconfigs).returns(false)
+            @resource1 = stub_everything 'res1'
+            @resource2 = stub_everything 'res2'
+            @resources = [ @resource1, @resource2 ]
+
+            @dbresource1 = stub_everything 'dbres1'
+            @dbresource2 = stub_everything 'dbres2'
+            @dbresources = { 1 => @dbresource1, 2 => @dbresource2 }
+
+            @host = Puppet::Rails::Host.new(:name => "foo", :environment => "production", :ip => "127.0.0.1")
+            @host.stubs(:find_resources).returns(@dbresources)
+            @host.stubs(:find_resources_parameters_tags)
+            @host.stubs(:compare_to_catalog)
+            @host.stubs(:id).returns(1)
+        end
+
+        it "should find all database resources" do
+            @host.expects(:find_resources)
+
+            @host.merge_resources(@resources)
+        end
+
+        it "should find all paramaters and tags for those database resources" do
+            @host.expects(:find_resources_parameters_tags).with(@dbresources)
+
+            @host.merge_resources(@resources)
+        end
+
+        it "should compare all database resources to catalog" do
+            @host.expects(:compare_to_catalog).with(@dbresources, @resources)
+
+            @host.merge_resources(@resources)
+        end
+
+        it "should compare only exported resources in thin_storeconfigs mode" do
+            Puppet.settings.stubs(:[]).with(:thin_storeconfigs).returns(true)
+            @resource1.stubs(:exported?).returns(true)
+
+            @host.expects(:compare_to_catalog).with(@dbresources, [ @resource1 ])
+
+            @host.merge_resources(@resources)
+        end
+    end
+
+    describe "when searching the database for host resources" do
+        before :each do
+            Puppet.settings.stubs(:[]).with(:thin_storeconfigs).returns(false)
+            @resource1 = stub_everything 'res1', :id => 1
+            @resource2 = stub_everything 'res2', :id => 2
+            @resources = [ @resource1, @resource2 ]
+
+            @dbresources = stub 'resources'
+            @dbresources.stubs(:find).returns(@resources)
+
+            @host = Puppet::Rails::Host.new(:name => "foo", :environment => "production", :ip => "127.0.0.1")
+            @host.stubs(:resources).returns(@dbresources)
+        end
+
+        it "should return a hash keyed by id of all resources" do
+            @host.find_resources.should == { 1 => @resource1, 2 => @resource2 }
+        end
+
+        it "should return a hash keyed by id of only exported resources in thin_storeconfigs mode" do
+            Puppet.settings.stubs(:[]).with(:thin_storeconfigs).returns(true)
+            @dbresources.expects(:find).with { |*h| h[1][:conditions] == { :exported => true } }.returns([])
+
+            @host.find_resources
+        end
+    end
 end