diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/puppet/defaults.rb | 8 | ||||
| -rw-r--r-- | lib/puppet/indirector/rest.rb | 12 | ||||
| -rw-r--r-- | lib/puppet/network/http/compression.rb | 112 | ||||
| -rwxr-xr-x | lib/puppet/type/file/content.rb | 8 |
4 files changed, 132 insertions, 8 deletions
diff --git a/lib/puppet/defaults.rb b/lib/puppet/defaults.rb index 2e3888298..b0995c2a9 100644 --- a/lib/puppet/defaults.rb +++ b/lib/puppet/defaults.rb @@ -621,7 +621,13 @@ module Puppet :graph => [false, "Whether to create dot graph files for the different configuration graphs. These dot files can be interpreted by tools like OmniGraffle or dot (which is part of ImageMagick)."], - :graphdir => ["$statedir/graphs", "Where to store dot-outputted graphs."] + :graphdir => ["$statedir/graphs", "Where to store dot-outputted graphs."], + :http_compression => [false, "Allow http compression in REST communication with the master. + This setting might improve performance for puppetd -> puppetmasterd communications over slow WANs. + Your puppetmaster needs to support compression (usually by activating some settings in a reverse-proxy + in front of the puppetmaster, which rules out webrick). + It is harmless to activate this settings if your master doesn't support + compression, but if it supports it, this setting might reduce performance on high-speed LANs."] ) # Plugin information. diff --git a/lib/puppet/indirector/rest.rb b/lib/puppet/indirector/rest.rb index 4fd385919..f9b11c16f 100644 --- a/lib/puppet/indirector/rest.rb +++ b/lib/puppet/indirector/rest.rb @@ -3,10 +3,12 @@ require 'uri' require 'puppet/network/http_pool' require 'puppet/network/http/api/v1' +require 'puppet/network/http/compression' # Access objects via REST class Puppet::Indirector::REST < Puppet::Indirector::Terminus include Puppet::Network::HTTP::API::V1 + include Puppet::Network::HTTP::Compression.module class << self attr_reader :server_setting, :port_setting @@ -43,22 +45,24 @@ class Puppet::Indirector::REST < Puppet::Indirector::Terminus content_type = response['content-type'].gsub(/\s*;.*$/,'') # strip any appended charset + body = uncompress_body(response) + # Convert the response to a deserialized object. if multiple - model.convert_from_multiple(content_type, response.body) + model.convert_from_multiple(content_type, body) else - model.convert_from(content_type, response.body) + model.convert_from(content_type, body) end else # Raise the http error if we didn't get a 'success' of some kind. - message = "Error %s on SERVER: %s" % [response.code, (response.body||'').empty? ? response.message : response.body] + message = "Error %s on SERVER: %s" % [response.code, (response.body||'').empty? ? response.message : uncompress_body(response)] raise Net::HTTPError.new(message, response) end end # Provide appropriate headers. def headers - {"Accept" => model.supported_formats.join(", ")} + add_accept_encoding({"Accept" => model.supported_formats.join(", ")}) end def network(request) diff --git a/lib/puppet/network/http/compression.rb b/lib/puppet/network/http/compression.rb new file mode 100644 index 000000000..722ecfe24 --- /dev/null +++ b/lib/puppet/network/http/compression.rb @@ -0,0 +1,112 @@ +require 'puppet/network/http' + +module Puppet::Network::HTTP::Compression + + # this module function allows to use the right underlying + # methods depending on zlib presence + def module + return Active if Puppet.features.zlib? + return None + end + module_function :module + + module Active + require 'zlib' + require 'stringio' + + # return an uncompressed body if the response has been + # compressed + def uncompress_body(response) + case response['content-encoding'] + when 'gzip' + return Zlib::GzipReader.new(StringIO.new(response.body)).read + when 'deflate' + return Zlib::Inflate.new().inflate(response.body) + when nil, 'identity' + return response.body + else + raise Net::HTTPError.new("Unknown content encoding - #{response['content-encoding']}", response) + end + end + + def uncompress(response) + raise Net::HTTPError.new("No block passed") unless block_given? + + case response['content-encoding'] + when 'gzip','deflate' + uncompressor = ZlibAdapter.new + when nil, 'identity' + uncompressor = IdentityAdapter.new + else + raise Net::HTTPError.new("Unknown content encoding - #{response['content-encoding']}", response) + end + + yield uncompressor + + uncompressor.close + end + + def add_accept_encoding(headers={}) + headers['accept-encoding'] = 'gzip; q=1.0, deflate; q=1.0; identity' if Puppet.settings[:http_compression] + headers + end + + # This adapters knows how to uncompress both 'zlib' stream (the deflate algorithm from Content-Encoding) + # and GZip streams. + class ZlibAdapter + def initialize + # Create an inflater that knows to parse GZip streams and zlib streams. + # This uses a property of the C Zlib library, documented as follow: + # windowBits can also be greater than 15 for optional gzip decoding. Add + # 32 to windowBits to enable zlib and gzip decoding with automatic header + # detection, or add 16 to decode only the gzip format (the zlib format will + # return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is + # a crc32 instead of an adler32. + @uncompressor = Zlib::Inflate.new(15 + 32) + @first = true + end + + def uncompress(chunk) + out = @uncompressor.inflate(chunk) + @first = false + return out + rescue Zlib::DataError => z + # it can happen that we receive a raw deflate stream + # which might make our inflate throw a data error. + # in this case, we try with a verbatim (no header) + # deflater. + @uncompressor = Zlib::Inflate.new + retry if @first + raise + end + + def close + @uncompressor.finish + @uncompressor.close + end + end + end + + module None + def uncompress_body(response) + response.body + end + + def add_accept_encoding(headers) + headers + end + + def uncompress(response) + yield IdentityAdapter.new + end + end + + class IdentityAdapter + def uncompress(chunk) + chunk + end + + def close + end + end +end diff --git a/lib/puppet/type/file/content.rb b/lib/puppet/type/file/content.rb index 1817d5646..ff139f208 100755 --- a/lib/puppet/type/file/content.rb +++ b/lib/puppet/type/file/content.rb @@ -3,12 +3,14 @@ require 'uri' require 'puppet/util/checksums' require 'puppet/network/http/api/v1' +require 'puppet/network/http/compression' module Puppet Puppet::Type.type(:file).newproperty(:content) do include Puppet::Util::Diff include Puppet::Util::Checksums include Puppet::Network::HTTP::API::V1 + include Puppet::Network::HTTP::Compression.module attr_reader :actual_content @@ -159,13 +161,13 @@ module Puppet else request = Puppet::Indirector::Request.new(:file_content, :find, source_or_content.full_path) connection = Puppet::Network::HttpPool.http_instance(source_or_content.server, source_or_content.port) - connection.request_get(indirection2uri(request), {"Accept" => "raw"}) do |response| + connection.request_get(indirection2uri(request), add_accept_encoding({"Accept" => "raw"})) do |response| case response.code when "404"; nil - when /^2/; response.read_body { |chunk| yield chunk } + when /^2/; uncompress(response) { |uncompressor| response.read_body { |chunk| yield uncompressor.uncompress(chunk) } } else # Raise the http error if we didn't get a 'success' of some kind. - message = "Error %s on SERVER: %s" % [response.code, (response.body||'').empty? ? response.message : response.body] + message = "Error %s on SERVER: %s" % [response.code, (response.body||'').empty? ? response.message : uncompress_body(response)] raise Net::HTTPError.new(message, response) end end |