Adding caching of file metadata to the autoloader

The cache isn't actually used yet - this just adds
all of the plumbing.

It was found that stat'ing files that didn't exist
could take up to 85% of a run, so this is progress
toward getting rid of those stats.

Signed-off-by: Luke Kanies <luke@madstop.com>

4 files changed, 242 insertions, 3 deletions

diff --git a/lib/puppet/util/autoload.rb b/lib/puppet/util/autoload.rb
index 7c176779a..db06ea934 100644
--- a/lib/puppet/util/autoload.rb
+++ b/lib/puppet/util/autoload.rb
@@ -3,9 +3,12 @@ require 'puppet/util/cacher'
 
 # Autoload paths, either based on names or all at once.
 class Puppet::Util::Autoload
+    require 'puppet/util/autoload/file_cache'
+
     include Puppet::Util
     include Puppet::Util::Warnings
     include Puppet::Util::Cacher
+    include Puppet::Util::Autoload::FileCache
 
     @autoloaders = {}
     @loaded = []
@@ -74,7 +77,7 @@ class Puppet::Util::Autoload
 
         searchpath.each do |dir|
             file = File.join(dir, path)
-            next unless FileTest.exist?(file)
+            next unless file_exist?(file)
             begin
                 Kernel.load file, @wrap
                 name = symbolize(name)
diff --git a/lib/puppet/util/autoload/file_cache.rb b/lib/puppet/util/autoload/file_cache.rb
new file mode 100644
index 000000000..6de525a8d
--- /dev/null
+++ b/lib/puppet/util/autoload/file_cache.rb
@@ -0,0 +1,97 @@
+module Puppet::Util::Autoload::FileCache
+    @found_files = {}
+    @missing_files = {}
+    class << self
+        attr_reader :found_files, :missing_files
+    end
+
+    # Only used for testing.
+    def self.clear
+        @found_files.clear
+        @missing_files.clear
+    end
+
+    def found_files
+        Puppet::Util::Autoload::FileCache.found_files
+    end
+
+    def missing_files
+        Puppet::Util::Autoload::FileCache.missing_files
+    end
+
+    def directory_exist?(path)
+        cache = cached_data?(path, :directory?)
+        return cache unless cache.nil?
+
+        begin
+            stat = File.lstat(path)
+            if stat.directory?
+                found_file(path, stat)
+                return true
+            else
+                missing_file(path)
+                return false
+            end
+        rescue Errno::ENOENT
+            missing_file(path)
+            return false
+        end
+    end
+
+    def file_exist?(path)
+        cache = cached_data?(path)
+        return cache unless cache.nil?
+
+        begin
+            stat = File.lstat(path)
+            found_file(path, stat)
+            return true
+        rescue Errno::ENOENT
+            missing_file(path)
+            return false
+        end
+    end
+
+    def found_file?(path, type = nil)
+        if data = found_files[path] and ! data_expired?(data[:time])
+            if type and ! data[:stat].send(type)
+                return false
+            end
+            return true
+        else
+            return false
+        end
+    end
+
+    def found_file(path, stat)
+        found_files[path] = {:stat => stat, :time => Time.now}
+    end
+
+    def missing_file?(path)
+        if time = missing_files[path] and ! data_expired?(time)
+            return true
+        else
+            return false
+        end
+    end
+
+    def missing_file(path)
+        missing_files[path] = Time.now
+    end
+
+    private
+
+    def cached_data?(path, type = nil)
+        if found_file?(path, type)
+            return true
+        elsif missing_file?(path)
+            return false
+        else
+            return nil
+        end
+    end
+
+    def data_expired?(time)
+        Time.now - time > 15
+    end
+end
diff --git a/spec/unit/util/autoload.rb b/spec/unit/util/autoload.rb
index 53b90db69..3bd5b50ad 100755
--- a/spec/unit/util/autoload.rb
+++ b/spec/unit/util/autoload.rb
@@ -51,6 +51,10 @@ describe Puppet::Util::Autoload do
         end
     end
 
+    it "should include its FileCache module" do
+        Puppet::Util::Autoload.ancestors.should be_include(Puppet::Util::Autoload::FileCache)
+    end
+
     describe "when loading a file" do
         before do
             @autoload.stubs(:searchpath).returns %w{/a}
@@ -58,14 +62,20 @@ describe Puppet::Util::Autoload do
 
         [RuntimeError, LoadError, SyntaxError].each do |error|
             it "should not die an if a #{error.to_s} exception is thrown" do
-                FileTest.stubs(:directory?).returns true
-                FileTest.stubs(:exist?).returns true
+                @autoload.stubs(:file_exist?).returns true
 
                 Kernel.expects(:load).raises error
 
                 @autoload.load("foo")
             end
         end
+
+        it "should skip files that it knows are missing" do
+            @autoload.expects(:missing_file?).returns true
+            @autoload.expects(:eachdir).never
+
+            @autoload.load("foo")
+        end
     end
 
     describe "when loading all files" do
diff --git a/spec/unit/util/autoload/file_cache.rb b/spec/unit/util/autoload/file_cache.rb
new file mode 100755
index 000000000..ad4e75f74
--- /dev/null
+++ b/spec/unit/util/autoload/file_cache.rb
@@ -0,0 +1,129 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../../../spec_helper'
+require 'puppet/util/autoload/file_cache'
+
+class FileCacheTester
+    include Puppet::Util::Autoload::FileCache
+end
+
+describe Puppet::Util::Autoload::FileCache do
+    before do
+        @cacher = FileCacheTester.new
+    end
+
+    after do
+        Puppet::Util::Autoload::FileCache.clear
+    end
+
+    describe "when checking whether files exist" do
+        it "should have a method for testing whether a file exists" do
+            @cacher.should respond_to(:file_exist?)
+        end
+
+        it "should use lstat to determine whether a file exists" do
+            File.expects(:lstat).with("/my/file")
+            @cacher.file_exist?("/my/file")
+        end
+
+        it "should consider a file as absent if its lstat fails" do
+            File.expects(:lstat).with("/my/file").raises Errno::ENOENT
+            @cacher.should_not be_file_exist("/my/file")
+        end
+
+        it "should consider a file as present if its lstat succeeds" do
+            File.expects(:lstat).with("/my/file").returns mock("stat")
+            @cacher.should be_file_exist("/my/file")
+        end
+
+        it "should not stat a file twice in quick succession when the file is missing" do
+            File.expects(:lstat).with("/my/file").once.raises Errno::ENOENT
+            @cacher.should_not be_file_exist("/my/file")
+            @cacher.should_not be_file_exist("/my/file")
+        end
+
+        it "should not stat a file twice in quick succession when the file is present" do
+            File.expects(:lstat).with("/my/file").once.returns mock("stat")
+            @cacher.should be_file_exist("/my/file")
+            @cacher.should be_file_exist("/my/file")
+        end
+
+        it "should expire cached data after 15 seconds" do
+            now = Time.now
+
+            later = now + 16
+
+            Time.expects(:now).times(3).returns(now).then.returns(later).then.returns(later)
+            File.expects(:lstat).with("/my/file").times(2).returns(mock("stat")).then.raises Errno::ENOENT
+            @cacher.should be_file_exist("/my/file")
+            @cacher.should_not be_file_exist("/my/file")
+        end
+
+        it "should share cached data across autoload instances" do
+            File.expects(:lstat).with("/my/file").once.returns mock("stat")
+            other = Puppet::Util::Autoload.new("bar", "tmp")
+
+            @cacher.should be_file_exist("/my/file")
+            other.should be_file_exist("/my/file")
+        end
+    end
+
+    describe "when checking whether files exist" do
+        before do
+            @stat = stub 'stat', :directory? => true
+        end
+
+        it "should have a method for determining whether a directory exists" do
+            @cacher.should respond_to(:directory_exist?)
+        end
+
+        it "should use lstat to determine whether a directory exists" do
+            File.expects(:lstat).with("/my/file").returns @stat
+            @cacher.directory_exist?("/my/file")
+        end
+
+        it "should consider a directory as absent if its lstat fails" do
+            File.expects(:lstat).with("/my/file").raises Errno::ENOENT
+            @cacher.should_not be_directory_exist("/my/file")
+        end
+
+        it "should consider a directory as present if its lstat succeeds and the stat is of a directory" do
+            @stat.expects(:directory?).returns true
+            File.expects(:lstat).with("/my/file").returns @stat
+            @cacher.should be_directory_exist("/my/file")
+        end
+
+        it "should consider a directory as absent if its lstat succeeds and the stat is not of a directory" do
+            @stat.expects(:directory?).returns false
+            File.expects(:lstat).with("/my/file").returns @stat
+            @cacher.should_not be_directory_exist("/my/file")
+        end
+
+        it "should not stat a directory twice in quick succession when the file is missing" do
+            File.expects(:lstat).with("/my/file").once.raises Errno::ENOENT
+            @cacher.should_not be_directory_exist("/my/file")
+            @cacher.should_not be_directory_exist("/my/file")
+        end
+
+        it "should not stat a directory twice in quick succession when the file is present" do
+            File.expects(:lstat).with("/my/file").once.returns @stat
+            @cacher.should be_directory_exist("/my/file")
+            @cacher.should be_directory_exist("/my/file")
+        end
+
+        it "should not consider a file to be a directory based on cached data" do
+            @stat.stubs(:directory?).returns false
+            File.stubs(:lstat).with("/my/file").returns @stat
+            @cacher.file_exist?("/my/file")
+            @cacher.should_not be_directory_exist("/my/file")
+        end
+
+        it "should share cached data across autoload instances" do
+            File.expects(:lstat).with("/my/file").once.returns @stat
+            other = Puppet::Util::Autoload.new("bar", "tmp")
+
+            @cacher.should be_directory_exist("/my/file")
+            other.should be_directory_exist("/my/file")
+        end
+    end
+end