Fix #3408 - enable puppetd http compression

This patch adds HTTP response decompression (both gzip and deflate streams).
This feature is disabled by default, and enabled with --http_compression.
This feature can be activated only if the local ruby version supports the
zlib ruby extension.

HTTP response decompression is active for all REST communications and file
sourcing.

To enable http compression on the server side, it is needed to use a
reverse proxy like Apache or Nginx with adhoc configuration:

Nginx:
gzip on;
gzip_types text/pson text/json text/marshall text/yaml application/x-raw text/plain;

Apache:
LoadModule deflate_module /usr/lib/apache2/modules/mod_deflate.so
AddOutputFilterByType DEFLATE text/plain text/pson text/json text/marshall text/yaml application/x-raw

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

4 files changed, 132 insertions, 8 deletions

diff --git a/lib/puppet/defaults.rb b/lib/puppet/defaults.rb
index 2e3888298..b0995c2a9 100644
--- a/lib/puppet/defaults.rb
+++ b/lib/puppet/defaults.rb
@@ -621,7 +621,13 @@ module Puppet
         :graph => [false, "Whether to create dot graph files for the different
             configuration graphs.  These dot files can be interpreted by tools
             like OmniGraffle or dot (which is part of ImageMagick)."],
-        :graphdir => ["$statedir/graphs", "Where to store dot-outputted graphs."]
+        :graphdir => ["$statedir/graphs", "Where to store dot-outputted graphs."],
+        :http_compression => [false, "Allow http compression in REST communication with the master.
+            This setting might improve performance for puppetd -> puppetmasterd communications over slow WANs.
+            Your puppetmaster needs to support compression (usually by activating some settings in a reverse-proxy
+            in front of the puppetmaster, which rules out webrick). 
+            It is harmless to activate this settings if your master doesn't support
+            compression, but if it supports it, this setting might reduce performance on high-speed LANs."]
     )
 
     # Plugin information.
diff --git a/lib/puppet/indirector/rest.rb b/lib/puppet/indirector/rest.rb
index 4fd385919..f9b11c16f 100644
--- a/lib/puppet/indirector/rest.rb
+++ b/lib/puppet/indirector/rest.rb
@@ -3,10 +3,12 @@ require 'uri'
 
 require 'puppet/network/http_pool'
 require 'puppet/network/http/api/v1'
+require 'puppet/network/http/compression'
 
 # Access objects via REST
 class Puppet::Indirector::REST < Puppet::Indirector::Terminus
     include Puppet::Network::HTTP::API::V1
+    include Puppet::Network::HTTP::Compression.module
 
     class << self
         attr_reader :server_setting, :port_setting
@@ -43,22 +45,24 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus
 
             content_type = response['content-type'].gsub(/\s*;.*$/,'') # strip any appended charset
 
+            body = uncompress_body(response)
+
             # Convert the response to a deserialized object.
             if multiple
-                model.convert_from_multiple(content_type, response.body)
+                model.convert_from_multiple(content_type, body)
             else
-                model.convert_from(content_type, response.body)
+                model.convert_from(content_type, body)
             end
         else
             # Raise the http error if we didn't get a 'success' of some kind.
-            message = "Error %s on SERVER: %s" % [response.code, (response.body||'').empty? ? response.message : response.body]
+            message = "Error %s on SERVER: %s" % [response.code, (response.body||'').empty? ? response.message : uncompress_body(response)]
             raise Net::HTTPError.new(message, response)
         end
     end
 
     # Provide appropriate headers.
     def headers
-        {"Accept" => model.supported_formats.join(", ")}
+        add_accept_encoding({"Accept" => model.supported_formats.join(", ")})
     end
 
     def network(request)
diff --git a/lib/puppet/network/http/compression.rb b/lib/puppet/network/http/compression.rb
new file mode 100644
index 000000000..722ecfe24
--- /dev/null
+++ b/lib/puppet/network/http/compression.rb
@@ -0,0 +1,112 @@
+require 'puppet/network/http'
+
+module Puppet::Network::HTTP::Compression
+
+    # this module function allows to use the right underlying
+    # methods depending on zlib presence
+    def module
+        return Active if Puppet.features.zlib?
+        return None
+    end
+    module_function :module
+
+    module Active
+        require 'zlib'
+        require 'stringio'
+
+        # return an uncompressed body if the response has been
+        # compressed
+        def uncompress_body(response)
+            case response['content-encoding']
+            when 'gzip'
+                return Zlib::GzipReader.new(StringIO.new(response.body)).read
+            when 'deflate'
+                return Zlib::Inflate.new().inflate(response.body)
+            when nil, 'identity'
+                return response.body
+            else
+                raise Net::HTTPError.new("Unknown content encoding - #{response['content-encoding']}", response)
+            end
+        end
+
+        def uncompress(response)
+            raise Net::HTTPError.new("No block passed") unless block_given?
+
+            case response['content-encoding']
+            when 'gzip','deflate'
+                uncompressor = ZlibAdapter.new
+            when nil, 'identity'
+                uncompressor = IdentityAdapter.new
+            else
+                raise Net::HTTPError.new("Unknown content encoding - #{response['content-encoding']}", response)
+            end
+
+            yield uncompressor
+
+            uncompressor.close
+        end
+
+        def add_accept_encoding(headers={})
+            headers['accept-encoding'] = 'gzip; q=1.0, deflate; q=1.0; identity' if Puppet.settings[:http_compression]
+            headers
+        end
+
+        # This adapters knows how to uncompress both 'zlib' stream (the deflate algorithm from Content-Encoding)
+        # and GZip streams.
+        class ZlibAdapter
+            def initialize
+                # Create an inflater that knows to parse GZip streams and zlib streams.
+                # This uses a property of the C Zlib library, documented as follow:
+                #   windowBits can also be greater than 15 for optional gzip decoding. Add
+                #   32 to windowBits to enable zlib and gzip decoding with automatic header
+                #   detection, or add 16 to decode only the gzip format (the zlib format will
+                #   return a Z_DATA_ERROR).  If a gzip stream is being decoded, strm->adler is
+                #   a crc32 instead of an adler32.
+                @uncompressor = Zlib::Inflate.new(15 + 32)
+                @first = true
+            end
+
+            def uncompress(chunk)
+                out = @uncompressor.inflate(chunk)
+                @first = false
+                return out
+            rescue Zlib::DataError => z
+                # it can happen that we receive a raw deflate stream
+                # which might make our inflate throw a data error.
+                # in this case, we try with a verbatim (no header)
+                # deflater.
+                @uncompressor = Zlib::Inflate.new
+                retry if @first
+                raise
+            end
+
+            def close
+                @uncompressor.finish
+                @uncompressor.close
+            end
+        end
+    end
+
+    module None
+        def uncompress_body(response)
+            response.body
+        end
+
+        def add_accept_encoding(headers)
+            headers
+        end
+
+        def uncompress(response)
+            yield IdentityAdapter.new
+        end
+    end
+
+    class IdentityAdapter
+        def uncompress(chunk)
+            chunk
+        end
+
+        def close
+        end
+    end
+end
diff --git a/lib/puppet/type/file/content.rb b/lib/puppet/type/file/content.rb
index 1817d5646..ff139f208 100755
--- a/lib/puppet/type/file/content.rb
+++ b/lib/puppet/type/file/content.rb
@@ -3,12 +3,14 @@ require 'uri'
 
 require 'puppet/util/checksums'
 require 'puppet/network/http/api/v1'
+require 'puppet/network/http/compression'
 
 module Puppet
     Puppet::Type.type(:file).newproperty(:content) do
         include Puppet::Util::Diff
         include Puppet::Util::Checksums
         include Puppet::Network::HTTP::API::V1
+        include Puppet::Network::HTTP::Compression.module
 
         attr_reader :actual_content
 
@@ -159,13 +161,13 @@ module Puppet
             else
                 request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path)
                 connection = Puppet::Network::HttpPool.http_instance(source_or_content.server, source_or_content.port)
-                connection.request_get(indirection2uri(request), {"Accept" => "raw"}) do |response|
+                connection.request_get(indirection2uri(request), add_accept_encoding({"Accept" => "raw"})) do |response|
                     case response.code
                     when "404"; nil
-                    when /^2/;  response.read_body { |chunk| yield chunk }
+                    when /^2/;  uncompress(response) { |uncompressor| response.read_body { |chunk| yield uncompressor.uncompress(chunk) } }
                     else
                         # Raise the http error if we didn't get a 'success' of some kind.
-                        message = "Error %s on SERVER: %s" % [response.code, (response.body||'').empty? ? response.message : response.body]
+                        message = "Error %s on SERVER: %s" % [response.code, (response.body||'').empty? ? response.message : uncompress_body(response)]
                         raise Net::HTTPError.new(message, response)
                     end
                 end