Fix #3408 - enable puppetd http compression

This patch adds HTTP response decompression (both gzip and deflate streams).
This feature is disabled by default, and enabled with --http_compression.
This feature can be activated only if the local ruby version supports the
zlib ruby extension.

HTTP response decompression is active for all REST communications and file
sourcing.

To enable http compression on the server side, it is needed to use a
reverse proxy like Apache or Nginx with adhoc configuration:

Nginx:
gzip on;
gzip_types text/pson text/json text/marshall text/yaml application/x-raw text/plain;

Apache:
LoadModule deflate_module /usr/lib/apache2/modules/mod_deflate.so
AddOutputFilterByType DEFLATE text/plain text/pson text/json text/marshall text/yaml application/x-raw

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

5 files changed, 288 insertions, 1 deletions

diff --git a/spec/integration/defaults.rb b/spec/integration/defaults.rb
index 621b5be16..bae281d72 100755
--- a/spec/integration/defaults.rb
+++ b/spec/integration/defaults.rb
@@ -240,4 +240,8 @@ describe "Puppet defaults" do
     it "should have a 'certificate_revocation' setting that defaults to true" do
         Puppet.settings[:certificate_revocation].should be_true
     end
+
+    it "should have an http_compression setting that defaults to false" do
+        Puppet.settings[:http_compression].should be_false
+    end
 end
diff --git a/spec/unit/indirector/certificate/rest.rb b/spec/unit/indirector/certificate/rest.rb
index a3257543b..9f272fbb2 100755
--- a/spec/unit/indirector/certificate/rest.rb
+++ b/spec/unit/indirector/certificate/rest.rb
@@ -47,6 +47,7 @@ rn/G
 
         response = stub 'response', :code => "200", :body => cert_string
         response.stubs(:[]).with('content-type').returns "text/plain"
+        response.stubs(:[]).with('content-encoding')
         network.expects(:get).returns response
 
         request = Puppet::Indirector::Request.new(:certificate, :find, "foo.com")
diff --git a/spec/unit/indirector/rest.rb b/spec/unit/indirector/rest.rb
index 1cb34c4f4..668158c53 100755
--- a/spec/unit/indirector/rest.rb
+++ b/spec/unit/indirector/rest.rb
@@ -41,6 +41,7 @@ describe Puppet::Indirector::REST do
 
         @response = stub('mock response', :body => 'result', :code => "200")
         @response.stubs(:[]).with('content-type').returns "text/plain"
+        @response.stubs(:[]).with('content-encoding').returns nil
 
         @searcher = @rest_class.new
         @searcher.stubs(:model).returns @model
@@ -97,6 +98,7 @@ describe Puppet::Indirector::REST do
 
                     @response = mock 'response'
                     @response.stubs(:code).returns rc.to_s
+                    @response.stubs(:[]).with('content-encoding').returns nil
                     @response.stubs(:message).returns "There was a problem (header)"
                 end
 
@@ -119,14 +121,23 @@ describe Puppet::Indirector::REST do
                     @response.stubs(:body).returns nil
                     lambda { @searcher.deserialize(@response) }.should raise_error(Net::HTTPError,"Error #{rc} on SERVER: There was a problem (header)")
                 end
+
+                describe "and with http compression" do
+                    it "should uncompress the body" do
+                        @response.stubs(:body).returns("compressed body")
+                        @searcher.expects(:uncompress_body).with(@response).returns("uncompressed")
+                        lambda { @searcher.deserialize(@response) }.should raise_error { |e| e.message =~ /uncompressed/ }
+                    end
+                end
             end
-        }    
+        }
 
         it "should return the results of converting from the format specified by the content-type header if the response code is in the 200s" do
             @model.expects(:convert_from).with("myformat", "mydata").returns "myobject"
 
             response = mock 'response'
             response.stubs(:[]).with("content-type").returns "myformat"
+            response.stubs(:[]).with("content-encoding").returns nil
             response.stubs(:body).returns "mydata"
             response.stubs(:code).returns "200"
 
@@ -138,6 +149,7 @@ describe Puppet::Indirector::REST do
 
             response = mock 'response'
             response.stubs(:[]).with("content-type").returns "myformat"
+            response.stubs(:[]).with("content-encoding").returns nil
             response.stubs(:body).returns "mydata"
             response.stubs(:code).returns "200"
 
@@ -149,11 +161,25 @@ describe Puppet::Indirector::REST do
 
             response = mock 'response'
             response.stubs(:[]).with("content-type").returns "text/plain; charset=utf-8"
+            response.stubs(:[]).with("content-encoding").returns nil
             response.stubs(:body).returns "mydata"
             response.stubs(:code).returns "200"
 
             @searcher.deserialize(response)
         end
+
+        it "should uncompress the body" do
+            @model.expects(:convert_from).with("myformat", "uncompressed mydata").returns "myobject"
+
+            response = mock 'response'
+            response.stubs(:[]).with("content-type").returns "myformat"
+            response.stubs(:body).returns "compressed mydata"
+            response.stubs(:code).returns "200"
+
+            @searcher.expects(:uncompress_body).with(response).returns("uncompressed mydata")
+
+            @searcher.deserialize(response).should == "myobject"
+        end
     end
 
     describe "when creating an HTTP client" do
@@ -232,6 +258,13 @@ describe Puppet::Indirector::REST do
             @searcher.find(@request)
         end
 
+        it "should add Accept-Encoding header" do
+            @searcher.expects(:add_accept_encoding).returns({"accept-encoding" => "gzip"})
+
+            @connection.expects(:get).with { |path, args| args["accept-encoding"] == "gzip" }.returns(@response)
+            @searcher.find(@request)
+        end
+
         it "should deserialize and return the network response" do
             @searcher.expects(:deserialize).with(@response).returns @instance
             @searcher.find(@request).should equal(@instance)
diff --git a/spec/unit/network/http/compression.rb b/spec/unit/network/http/compression.rb
new file mode 100644
index 000000000..9b9a2a516
--- /dev/null
+++ b/spec/unit/network/http/compression.rb
@@ -0,0 +1,199 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../../../spec_helper'
+
+describe "http compression" do
+
+    describe "when zlib is not available" do
+        before(:each) do
+            Puppet.features.stubs(:zlib?).returns false
+
+            require 'puppet/network/http/compression'
+            class HttpUncompressor
+                include Puppet::Network::HTTP::Compression::None
+            end
+
+            @uncompressor = HttpUncompressor.new
+        end
+
+        it "should have a module function that returns the None underlying module" do
+            Puppet::Network::HTTP::Compression.module.should == Puppet::Network::HTTP::Compression::None
+        end
+
+        it "should not add any Accept-Encoding header" do
+            @uncompressor.add_accept_encoding({}).should == {}
+        end
+
+        it "should not tamper the body" do
+            response = stub 'response', :body => "data"
+            @uncompressor.uncompress_body(response).should == "data"
+        end
+
+        it "should yield an identity uncompressor" do
+            response = stub 'response'
+            @uncompressor.uncompress(response) { |u|
+                u.should be_instance_of Puppet::Network::HTTP::Compression::IdentityAdapter
+            }
+        end
+    end
+
+    describe "when zlib is available" do
+        confine "Zlib is missing" => Puppet.features.zlib?
+
+        before(:each) do
+            Puppet.features.stubs(:zlib?).returns true
+
+            require 'puppet/network/http/compression'
+            class HttpUncompressor
+                include Puppet::Network::HTTP::Compression::Active
+            end
+
+            @uncompressor = HttpUncompressor.new
+        end
+
+        it "should have a module function that returns the Active underlying module" do
+            Puppet::Network::HTTP::Compression.module.should == Puppet::Network::HTTP::Compression::Active
+        end
+
+        it "should add an Accept-Encoding header when http compression is available" do
+            Puppet.settings.expects(:[]).with(:http_compression).returns(true)
+            headers = @uncompressor.add_accept_encoding({})
+            headers.should have_key('accept-encoding')
+            headers['accept-encoding'].should =~ /gzip/
+            headers['accept-encoding'].should =~ /deflate/
+            headers['accept-encoding'].should =~ /identity/
+        end
+
+        it "should not add Accept-Encoding header if http compression is not available" do
+            Puppet.settings.stubs(:[]).with(:http_compression).returns(false)
+            @uncompressor.add_accept_encoding({}).should == {}
+        end
+
+        describe "when uncompressing response body" do
+            before do
+                @response = stub 'response'
+                @response.stubs(:[]).with('content-encoding')
+                @response.stubs(:body).returns("mydata")
+            end
+
+            it "should return untransformed response body with no content-encoding" do
+                @uncompressor.uncompress_body(@response).should == "mydata"
+            end
+
+            it "should return untransformed response body with 'identity' content-encoding" do
+                @response.stubs(:[]).with('content-encoding').returns('identity')
+                @uncompressor.uncompress_body(@response).should == "mydata"
+            end
+
+            it "should use a Zlib inflater with 'deflate' content-encoding" do
+                @response.stubs(:[]).with('content-encoding').returns('deflate')
+
+                inflater = stub 'inflater'
+                Zlib::Inflate.expects(:new).returns(inflater)
+                inflater.expects(:inflate).with("mydata").returns "uncompresseddata"
+
+                @uncompressor.uncompress_body(@response).should == "uncompresseddata"
+            end
+
+            it "should use a GzipReader with 'gzip' content-encoding" do
+                @response.stubs(:[]).with('content-encoding').returns('gzip')
+
+                io = stub 'io'
+                StringIO.expects(:new).with("mydata").returns io
+
+                reader = stub 'gzip reader'
+                Zlib::GzipReader.expects(:new).with(io).returns(reader)
+                reader.expects(:read).returns "uncompresseddata"
+
+                @uncompressor.uncompress_body(@response).should == "uncompresseddata"
+            end
+        end
+
+        describe "when uncompressing by chunk" do
+            before do
+                @response = stub 'response'
+                @response.stubs(:[]).with('content-encoding')
+
+                @inflater = stub_everything 'inflater'
+                Zlib::Inflate.stubs(:new).returns(@inflater)
+            end
+
+            it "should yield an identity uncompressor with no content-encoding" do
+                @uncompressor.uncompress(@response) { |u|
+                    u.should be_instance_of Puppet::Network::HTTP::Compression::IdentityAdapter
+                }
+            end
+
+            it "should yield an identity uncompressor with 'identity' content-encoding" do
+                @response.stubs(:[]).with('content-encoding').returns 'identity'
+                @uncompressor.uncompress(@response) { |u|
+                    u.should be_instance_of Puppet::Network::HTTP::Compression::IdentityAdapter
+                }
+            end
+
+            %w{gzip deflate}.each do |c|
+                it "should yield a Zlib uncompressor with '#{c}' content-encoding" do
+                    @response.stubs(:[]).with('content-encoding').returns c
+                    @uncompressor.uncompress(@response) { |u|
+                        u.should be_instance_of Puppet::Network::HTTP::Compression::Active::ZlibAdapter
+                    }
+                end
+            end
+
+            it "should close the underlying adapter" do
+                adapter = stub_everything 'adapter'
+                Puppet::Network::HTTP::Compression::IdentityAdapter.expects(:new).returns(adapter)
+
+                adapter.expects(:close)
+                @uncompressor.uncompress(@response) { |u| }
+            end
+        end
+
+        describe "zlib adapter" do
+            before do
+                @inflater = stub_everything 'inflater'
+                Zlib::Inflate.stubs(:new).returns(@inflater)
+                @adapter = Puppet::Network::HTTP::Compression::Active::ZlibAdapter.new
+            end
+
+            it "should initialize the underlying inflater with gzip/zlib header parsing" do
+                Zlib::Inflate.expects(:new).with(15+32)
+                Puppet::Network::HTTP::Compression::Active::ZlibAdapter.new
+            end
+
+            it "should inflate the given chunk" do
+                @inflater.expects(:inflate).with("chunk")
+                @adapter.uncompress("chunk")
+            end
+
+            it "should return the inflated chunk" do
+                @inflater.stubs(:inflate).with("chunk").returns("uncompressed")
+                @adapter.uncompress("chunk").should == "uncompressed"
+            end
+
+            it "should try a 'regular' inflater on Zlib::DataError" do
+                @inflater.expects(:inflate).raises(Zlib::DataError.new "not a zlib stream")
+                inflater = stub_everything 'inflater2'
+                inflater.expects(:inflate).with("chunk").returns("uncompressed")
+                Zlib::Inflate.expects(:new).with().returns(inflater)
+                @adapter.uncompress("chunk")
+            end
+
+            it "should raise the error the second time" do
+                @inflater.expects(:inflate).raises(Zlib::DataError.new "not a zlib stream")
+                Zlib::Inflate.expects(:new).with().returns(@inflater)
+                lambda { @adapter.uncompress("chunk") }.should raise_error
+            end
+
+            it "should finish the stream on close" do
+                @inflater.expects(:finish)
+                @adapter.close
+            end
+
+            it "should close the stream on close" do
+                @inflater.expects(:close)
+                @adapter.close
+            end
+        end
+    end
+end
diff --git a/spec/unit/type/file/content.rb b/spec/unit/type/file/content.rb
index 96f037f7d..2289215bb 100755
--- a/spec/unit/type/file/content.rb
+++ b/spec/unit/type/file/content.rb
@@ -381,6 +381,56 @@ describe content do
                 @sum.expects(:sum_stream).yields(@digest).returns("checksum")
                 @content.write(@fh).should == "checksum"
             end
+
+            it "should get the current accept encoding header value" do
+                @content.expects(:add_accept_encoding)
+                @content.write(@fh)
+            end
+
+            it "should uncompress body on error" do
+                @response.expects(:code).returns("500")
+                @response.expects(:body).returns("compressed body")
+                @content.expects(:uncompress_body).with(@response).returns("uncompressed")
+                lambda { @content.write(@fh) }.should raise_error { |e| e.message =~ /uncompressed/ }
+            end
+
+            it "should uncompress chunk by chunk" do
+                uncompressor = stub_everything 'uncompressor'
+                @content.expects(:uncompress).with(@response).yields(uncompressor)
+                @response.expects(:code).returns("200")
+                @response.expects(:read_body).multiple_yields("chunk1","chunk2")
+
+                uncompressor.expects(:uncompress).with("chunk1").then.with("chunk2")
+                @content.write(@fh)
+            end
+
+            it "should write uncompressed chunks to the file" do
+                uncompressor = stub_everything 'uncompressor'
+                @content.expects(:uncompress).with(@response).yields(uncompressor)
+                @response.expects(:code).returns("200")
+                @response.expects(:read_body).multiple_yields("chunk1","chunk2")
+
+                uncompressor.expects(:uncompress).with("chunk1").returns("uncompressed1")
+                uncompressor.expects(:uncompress).with("chunk2").returns("uncompressed2")
+
+                @fh.expects(:print).with("uncompressed1")
+                @fh.expects(:print).with("uncompressed2")
+
+                @content.write(@fh)
+            end
+
+            it "should pass each uncompressed chunk to the current sum stream" do
+                uncompressor = stub_everything 'uncompressor'
+                @content.expects(:uncompress).with(@response).yields(uncompressor)
+                @response.expects(:code).returns("200")
+                @response.expects(:read_body).multiple_yields("chunk1","chunk2")
+
+                uncompressor.expects(:uncompress).with("chunk1").returns("uncompressed1")
+                uncompressor.expects(:uncompress).with("chunk2").returns("uncompressed2")
+
+                @digest.expects(:<<).with("uncompressed1").then.with("uncompressed2")
+                @content.write(@fh)
+            end
         end
     end
 end