Add master side file content streaming

This patch allows the puppetmaster to serve file chunks by chunks without
ever reading the file content in RAM.
This allows serving large files directly with the master without impacting
the master memory footprint.

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

8 files changed, 144 insertions, 10 deletions

diff --git a/lib/puppet/file_serving/content.rb b/lib/puppet/file_serving/content.rb
index 9fc1d08b3..87ef4fbbf 100644
--- a/lib/puppet/file_serving/content.rb
+++ b/lib/puppet/file_serving/content.rb
@@ -26,10 +26,11 @@ class Puppet::FileServing::Content < Puppet::FileServing::Base
         instance
     end
 
-    # Collect our data.
+    # BF: we used to fetch the file content here, but this is counter-productive
+    # for puppetmaster streaming of file content. So collect just returns itself
     def collect
         return if stat.ftype == "directory"
-        content
+        self
     end
 
     # Read the content of our file in.
@@ -44,6 +45,6 @@ class Puppet::FileServing::Content < Puppet::FileServing::Base
     end
 
     def to_raw
-        content
+        File.new(full_path(), "r")
     end
 end
diff --git a/lib/puppet/network/http/mongrel/rest.rb b/lib/puppet/network/http/mongrel/rest.rb
index 7b28d880b..8668bf802 100644
--- a/lib/puppet/network/http/mongrel/rest.rb
+++ b/lib/puppet/network/http/mongrel/rest.rb
@@ -54,12 +54,19 @@ class Puppet::Network::HTTP::MongrelREST < Mongrel::HttpHandler
         # we have a failure, unless we're on a version of mongrel that doesn't
         # support this.
         if status < 300
-            response.start(status) { |head, body| body.write(result) }
+            unless result.is_a?(File)
+                response.start(status) { |head, body| body.write(result) }
+            else
+                response.start(status) { |head, body| }
+                response.send_status(result.stat.size)
+                response.send_header
+                response.send_file(result.path)
+            end
         else
             begin
                 response.start(status,false,result) { |head, body| body.write(result) }
             rescue ArgumentError
-	        response.start(status)              { |head, body| body.write(result) }
+                response.start(status)              { |head, body| body.write(result) }
             end
         end
     end
diff --git a/lib/puppet/network/http/rack/rest.rb b/lib/puppet/network/http/rack/rest.rb
index 104751271..5245275dd 100644
--- a/lib/puppet/network/http/rack/rest.rb
+++ b/lib/puppet/network/http/rack/rest.rb
@@ -8,6 +8,24 @@ class Puppet::Network::HTTP::RackREST < Puppet::Network::HTTP::RackHttpHandler
     HEADER_ACCEPT = 'HTTP_ACCEPT'.freeze
     ContentType = 'Content-Type'.freeze
 
+    CHUNK_SIZE = 8192
+
+    class RackFile
+        def initialize(file)
+            @file = file
+        end
+
+        def each
+            while chunk = @file.read(CHUNK_SIZE)
+                yield chunk
+            end
+        end
+
+        def close
+            @file.close
+        end
+    end
+
     def initialize(args={})
         super()
         initialize_for_puppet(args)
@@ -20,7 +38,12 @@ class Puppet::Network::HTTP::RackREST < Puppet::Network::HTTP::RackHttpHandler
     # produce the body of the response
     def set_response(response, result, status = 200)
         response.status = status
-        response.write result
+        unless result.is_a?(File)
+            response.write result
+        else
+            response["Content-Length"] = result.stat.size
+            response.body = RackFile.new(result)
+        end
     end
 
     # Retrieve the accept header from the http request.
diff --git a/lib/puppet/network/http/webrick/rest.rb b/lib/puppet/network/http/webrick/rest.rb
index 274665dcd..9bfd0ce6a 100644
--- a/lib/puppet/network/http/webrick/rest.rb
+++ b/lib/puppet/network/http/webrick/rest.rb
@@ -50,7 +50,12 @@ class Puppet::Network::HTTP::WEBrickREST < WEBrick::HTTPServlet::AbstractServlet
 
     def set_response(response, result, status = 200)
         response.status = status
-        response.body          = result if status >= 200 and status != 304
+        if status >= 200 and status != 304
+            response.body = result
+            if result.is_a?(File)
+                response["content-length"] = result.stat.size
+            end
+        end
         response.reason_phrase = result if status < 200 or status >= 300
     end
 
diff --git a/spec/unit/file_serving/content.rb b/spec/unit/file_serving/content.rb
index eacaff89f..cba703902 100755
--- a/spec/unit/file_serving/content.rb
+++ b/spec/unit/file_serving/content.rb
@@ -25,15 +25,15 @@ describe Puppet::FileServing::Content do
         Puppet::FileServing::Content.new("/path").should respond_to(:collect)
     end
 
-    it "should retrieve and store its contents when its attributes are collected if the file is a normal file" do
+    it "should not retrieve and store its contents when its attributes are collected if the file is a normal file" do
         content = Puppet::FileServing::Content.new("/path")
 
         result = "foo"
         File.stubs(:lstat).returns(stub("stat", :ftype => "file"))
-        File.expects(:read).with("/path").returns result
+        File.expects(:read).with("/path").never
         content.collect
 
-        content.instance_variable_get("@content").should_not be_nil
+        content.instance_variable_get("@content").should be_nil
     end
 
     it "should not attempt to retrieve its contents if the file is a directory" do
@@ -70,6 +70,14 @@ describe Puppet::FileServing::Content do
 
         Puppet::FileServing::Content.from_raw("foo/bar").should equal(instance)
     end
+
+    it "should return an opened File when converted to raw" do
+        content = Puppet::FileServing::Content.new("/path")
+
+        File.expects(:new).with("/path","r").returns :file
+
+        content.to_raw.should == :file
+    end
 end
 
 describe Puppet::FileServing::Content, "when returning the contents" do
diff --git a/spec/unit/network/http/mongrel/rest.rb b/spec/unit/network/http/mongrel/rest.rb
index fb7e7e57d..55b6172d3 100755
--- a/spec/unit/network/http/mongrel/rest.rb
+++ b/spec/unit/network/http/mongrel/rest.rb
@@ -81,6 +81,23 @@ describe "Puppet::Network::HTTP::MongrelREST" do
                 @handler.set_response(@response, "mybody", 200)
             end
 
+            describe "when the result is a File" do
+                it "should use response send_file" do
+                    head = mock 'head'
+                    body = mock 'body'
+                    stat = stub 'stat', :size => 100
+                    file = stub 'file', :stat => stat, :path => "/tmp/path"
+                    file.stubs(:is_a?).with(File).returns(true)
+
+                    @response.expects(:start).with(200).yields(head, body)
+                    @response.expects(:send_status).with(100)
+                    @response.expects(:send_header)
+                    @response.expects(:send_file).with("/tmp/path")
+
+                    @handler.set_response(@response, file, 200)
+                end
+            end
+
             it "should set the status and reason and write the body when setting the response for a successful request" do
                 head = mock 'head'
                 body = mock 'body'
diff --git a/spec/unit/network/http/rack/rest.rb b/spec/unit/network/http/rack/rest.rb
index e916712f3..75642f9f7 100755
--- a/spec/unit/network/http/rack/rest.rb
+++ b/spec/unit/network/http/rack/rest.rb
@@ -74,6 +74,26 @@ describe "Puppet::Network::HTTP::RackREST" do
 
                 @handler.set_response(@response, "mybody", 400)
             end
+
+            describe "when result is a File" do
+                before :each do
+                    stat = stub 'stat', :size => 100
+                    @file = stub 'file', :stat => stat, :path => "/tmp/path"
+                    @file.stubs(:is_a?).with(File).returns(true)
+                end
+
+                it "should set the Content-Length header" do
+                    @response.expects(:[]=).with("Content-Length", 100)
+
+                    @handler.set_response(@response, @file, 200)
+                end
+
+                it "should return a RackFile adapter as body" do
+                    @response.expects(:body=).with { |val| val.is_a?(Puppet::Network::HTTP::RackREST::RackFile) }
+
+                    @handler.set_response(@response, @file, 200)
+                end
+            end
         end
 
         describe "and determining the request parameters" do
@@ -197,3 +217,33 @@ describe "Puppet::Network::HTTP::RackREST" do
         end
     end
 end
+
+describe Puppet::Network::HTTP::RackREST::RackFile do
+    before(:each) do
+        stat = stub 'stat', :size => 100
+        @file = stub 'file', :stat => stat, :path => "/tmp/path"
+        @rackfile = Puppet::Network::HTTP::RackREST::RackFile.new(@file)
+    end
+
+    it "should have an each method" do
+        @rackfile.should be_respond_to(:each)
+    end
+
+    it "should yield file chunks by chunks" do
+        @file.expects(:read).times(3).with(8192).returns("1", "2", nil)
+        i = 1
+        @rackfile.each do |chunk|
+            chunk.to_i.should == i
+            i += 1
+        end
+    end
+
+    it "should have a close method" do
+        @rackfile.should be_respond_to(:close)
+    end
+
+    it "should delegate close to File close" do
+        @file.expects(:close)
+        @rackfile.close
+    end
+end
\ No newline at end of file
diff --git a/spec/unit/network/http/webrick/rest.rb b/spec/unit/network/http/webrick/rest.rb
index f5c563e0d..f726fd9a7 100755
--- a/spec/unit/network/http/webrick/rest.rb
+++ b/spec/unit/network/http/webrick/rest.rb
@@ -69,6 +69,29 @@ describe Puppet::Network::HTTP::WEBrickREST do
                 @handler.set_response(@response, "mybody", 200)
             end
 
+            describe "when the result is a File" do
+                before(:each) do
+                    stat = stub 'stat', :size => 100
+                    @file = stub 'file', :stat => stat, :path => "/tmp/path"
+                    @file.stubs(:is_a?).with(File).returns(true)
+                end
+
+                it "should serve it" do
+                    @response.stubs(:[]=)
+
+                    @response.expects(:status=).with 200
+                    @response.expects(:body=).with @file
+
+                    @handler.set_response(@response, @file, 200)
+                end
+
+                it "should set the Content-Length header" do
+                    @response.expects(:[]=).with('content-length', 100)
+
+                    @handler.set_response(@response, @file, 200)
+                end
+            end
+
             it "should set the status and message on the response when setting the response for a failed query" do
                 @response.expects(:status=).with 400
                 @response.expects(:reason_phrase=).with "mybody"