Add master side file content streaming

This patch allows the puppetmaster to serve file chunks by chunks without
ever reading the file content in RAM.
This allows serving large files directly with the master without impacting
the master memory footprint.

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

4 files changed, 101 insertions, 3 deletions

diff --git a/spec/unit/file_serving/content.rb b/spec/unit/file_serving/content.rb
index eacaff89f..cba703902 100755
--- a/spec/unit/file_serving/content.rb
+++ b/spec/unit/file_serving/content.rb
@@ -25,15 +25,15 @@ describe Puppet::FileServing::Content do
         Puppet::FileServing::Content.new("/path").should respond_to(:collect)
     end
 
-    it "should retrieve and store its contents when its attributes are collected if the file is a normal file" do
+    it "should not retrieve and store its contents when its attributes are collected if the file is a normal file" do
         content = Puppet::FileServing::Content.new("/path")
 
         result = "foo"
         File.stubs(:lstat).returns(stub("stat", :ftype => "file"))
-        File.expects(:read).with("/path").returns result
+        File.expects(:read).with("/path").never
         content.collect
 
-        content.instance_variable_get("@content").should_not be_nil
+        content.instance_variable_get("@content").should be_nil
     end
 
     it "should not attempt to retrieve its contents if the file is a directory" do
@@ -70,6 +70,14 @@ describe Puppet::FileServing::Content do
 
         Puppet::FileServing::Content.from_raw("foo/bar").should equal(instance)
     end
+
+    it "should return an opened File when converted to raw" do
+        content = Puppet::FileServing::Content.new("/path")
+
+        File.expects(:new).with("/path","r").returns :file
+
+        content.to_raw.should == :file
+    end
 end
 
 describe Puppet::FileServing::Content, "when returning the contents" do
diff --git a/spec/unit/network/http/mongrel/rest.rb b/spec/unit/network/http/mongrel/rest.rb
index fb7e7e57d..55b6172d3 100755
--- a/spec/unit/network/http/mongrel/rest.rb
+++ b/spec/unit/network/http/mongrel/rest.rb
@@ -81,6 +81,23 @@ describe "Puppet::Network::HTTP::MongrelREST" do
                 @handler.set_response(@response, "mybody", 200)
             end
 
+            describe "when the result is a File" do
+                it "should use response send_file" do
+                    head = mock 'head'
+                    body = mock 'body'
+                    stat = stub 'stat', :size => 100
+                    file = stub 'file', :stat => stat, :path => "/tmp/path"
+                    file.stubs(:is_a?).with(File).returns(true)
+
+                    @response.expects(:start).with(200).yields(head, body)
+                    @response.expects(:send_status).with(100)
+                    @response.expects(:send_header)
+                    @response.expects(:send_file).with("/tmp/path")
+
+                    @handler.set_response(@response, file, 200)
+                end
+            end
+
             it "should set the status and reason and write the body when setting the response for a successful request" do
                 head = mock 'head'
                 body = mock 'body'
diff --git a/spec/unit/network/http/rack/rest.rb b/spec/unit/network/http/rack/rest.rb
index e916712f3..75642f9f7 100755
--- a/spec/unit/network/http/rack/rest.rb
+++ b/spec/unit/network/http/rack/rest.rb
@@ -74,6 +74,26 @@ describe "Puppet::Network::HTTP::RackREST" do
 
                 @handler.set_response(@response, "mybody", 400)
             end
+
+            describe "when result is a File" do
+                before :each do
+                    stat = stub 'stat', :size => 100
+                    @file = stub 'file', :stat => stat, :path => "/tmp/path"
+                    @file.stubs(:is_a?).with(File).returns(true)
+                end
+
+                it "should set the Content-Length header" do
+                    @response.expects(:[]=).with("Content-Length", 100)
+
+                    @handler.set_response(@response, @file, 200)
+                end
+
+                it "should return a RackFile adapter as body" do
+                    @response.expects(:body=).with { |val| val.is_a?(Puppet::Network::HTTP::RackREST::RackFile) }
+
+                    @handler.set_response(@response, @file, 200)
+                end
+            end
         end
 
         describe "and determining the request parameters" do
@@ -197,3 +217,33 @@ describe "Puppet::Network::HTTP::RackREST" do
         end
     end
 end
+
+describe Puppet::Network::HTTP::RackREST::RackFile do
+    before(:each) do
+        stat = stub 'stat', :size => 100
+        @file = stub 'file', :stat => stat, :path => "/tmp/path"
+        @rackfile = Puppet::Network::HTTP::RackREST::RackFile.new(@file)
+    end
+
+    it "should have an each method" do
+        @rackfile.should be_respond_to(:each)
+    end
+
+    it "should yield file chunks by chunks" do
+        @file.expects(:read).times(3).with(8192).returns("1", "2", nil)
+        i = 1
+        @rackfile.each do |chunk|
+            chunk.to_i.should == i
+            i += 1
+        end
+    end
+
+    it "should have a close method" do
+        @rackfile.should be_respond_to(:close)
+    end
+
+    it "should delegate close to File close" do
+        @file.expects(:close)
+        @rackfile.close
+    end
+end
\ No newline at end of file
diff --git a/spec/unit/network/http/webrick/rest.rb b/spec/unit/network/http/webrick/rest.rb
index f5c563e0d..f726fd9a7 100755
--- a/spec/unit/network/http/webrick/rest.rb
+++ b/spec/unit/network/http/webrick/rest.rb
@@ -69,6 +69,29 @@ describe Puppet::Network::HTTP::WEBrickREST do
                 @handler.set_response(@response, "mybody", 200)
             end
 
+            describe "when the result is a File" do
+                before(:each) do
+                    stat = stub 'stat', :size => 100
+                    @file = stub 'file', :stat => stat, :path => "/tmp/path"
+                    @file.stubs(:is_a?).with(File).returns(true)
+                end
+
+                it "should serve it" do
+                    @response.stubs(:[]=)
+
+                    @response.expects(:status=).with 200
+                    @response.expects(:body=).with @file
+
+                    @handler.set_response(@response, @file, 200)
+                end
+
+                it "should set the Content-Length header" do
+                    @response.expects(:[]=).with('content-length', 100)
+
+                    @handler.set_response(@response, @file, 200)
+                end
+            end
+
             it "should set the status and message on the response when setting the response for a failed query" do
                 @response.expects(:status=).with 400
                 @response.expects(:reason_phrase=).with "mybody"