Significantly reworking the internals of the fileparsing code. It now

passes around an instance of a FileRecord, rather than just a hash, which I
think makes it much easier to understand.

Moved the sshkey parsed provider test to its own directory and made it better.

This work is all being done so I can move cron jobs to using providers
instead of the current unmaintainable state of affairs.



git-svn-id: https://reductivelabs.com/svn/puppet/trunk@2281 980ebf18-57e1-0310-9a29-db15c13687c0

1 files changed, 213 insertions, 106 deletions

diff --git a/test/util/fileparsing.rb b/test/util/fileparsing.rb
index 93f784d52..12343c38b 100755
--- a/test/util/fileparsing.rb
+++ b/test/util/fileparsing.rb
@@ -15,23 +15,26 @@ class TestUtilFileParsing < Test::Unit::TestCase
         include Puppet::Util::FileParsing
     end
 
-    def test_lines
-        parser = FParser.new
+    def setup
+        super
+        @parser = FParser.new
+    end
 
-        assert_equal("\n", parser.line_separator,
+    def test_lines
+        assert_equal("\n", @parser.line_separator,
             "Default separator was incorrect")
 
         {"\n" => ["one two\nthree four", "one two\nthree four\n"],
          "\t" => ["one two\tthree four", "one two\tthree four\t"],
         }.each do |sep, tests|
             assert_nothing_raised do
-                parser.line_separator = sep
+                @parser.line_separator = sep
             end
-            assert_equal(sep, parser.line_separator,
+            assert_equal(sep, @parser.line_separator,
                 "Did not set separator")
 
             tests.each do |test|
-                assert_equal(["one two", "three four"], parser.lines(test),
+                assert_equal(["one two", "three four"], @parser.lines(test),
                     "Incorrectly parsed %s" % test.inspect)
             end
         end
@@ -39,9 +42,7 @@ class TestUtilFileParsing < Test::Unit::TestCase
 
     # Make sure parse calls the appropriate methods or errors out
     def test_parse
-        parser = FParser.new
-
-        parser.meta_def(:parse_line) do |line|
+        @parser.meta_def(:parse_line) do |line|
             line.split(/\s+/)
         end
 
@@ -49,7 +50,7 @@ class TestUtilFileParsing < Test::Unit::TestCase
         should = [%w{one line}, %w{two line}]
         ret = nil
         assert_nothing_raised do
-            ret = parser.parse(text)
+            ret = @parser.parse(text)
         end
 
         assert_equal(should, ret)
@@ -57,80 +58,78 @@ class TestUtilFileParsing < Test::Unit::TestCase
 
     # Make sure we correctly handle different kinds of text lines.
     def test_text_line
-        parser = FParser.new
-
         comment = "# this is a comment"
 
         # Make sure it fails if no regex is passed
         assert_raise(ArgumentError) do
-            parser.text_line :comment
+            @parser.text_line :comment
         end
 
         # define a text matching comment record
         assert_nothing_raised do
-            parser.text_line :comment, :match => /^#/
+            @parser.text_line :comment, :match => /^#/
         end
 
         # Make sure it matches
         assert_nothing_raised do
             assert_equal({:record_type => :comment, :line => comment}, 
-                 parser.parse_line(comment))
+                 @parser.parse_line(comment))
         end
 
         # But not something else
         assert_nothing_raised do
-            assert_nil(parser.parse_line("some other text"))
+            assert_nil(@parser.parse_line("some other text"))
         end
 
         # Now define another type and make sure we get the right one back
         assert_nothing_raised do
-            parser.text_line :blank, :match => /^\s*$/
+            @parser.text_line :blank, :match => /^\s*$/
         end
 
         # The comment should still match
         assert_nothing_raised do
             assert_equal({:record_type => :comment, :line => comment}, 
-                 parser.parse_line(comment))
+                 @parser.parse_line(comment))
         end
 
         # As should our new line type
         assert_nothing_raised do
             assert_equal({:record_type => :blank, :line => ""}, 
-                 parser.parse_line(""))
+                 @parser.parse_line(""))
         end
 
     end
 
     def test_parse_line
-        parser = FParser.new
+        Puppet[:trace] = false
 
         comment = "# this is a comment"
 
         # Make sure it fails if we don't have any record types defined
         assert_raise(Puppet::DevError) do
-            parser.parse_line(comment)
+            @parser.parse_line(comment)
         end
 
         # Now define a text matching comment record
         assert_nothing_raised do
-            parser.text_line :comment, :match => /^#/
+            @parser.text_line :comment, :match => /^#/
         end
 
         # And make sure we can't define another one with the same name
         assert_raise(ArgumentError) do
-            parser.text_line :comment, :match => /^"/
+            @parser.text_line :comment, :match => /^"/
         end
 
         result = nil
         assert_nothing_raised("Did not parse text line") do
-            result = parser.parse_line comment
+            result = @parser.parse_line comment
         end
 
         assert_equal({:record_type => :comment, :line => comment}, result)
 
         # Make sure we just return nil on unmatched lines.
         assert_nothing_raised("Did not parse text line") do
-            result = parser.parse_line "No match for this"
+            result = @parser.parse_line "No match for this"
         end
 
         assert_nil(result, "Somehow matched an empty line")
@@ -138,63 +137,61 @@ class TestUtilFileParsing < Test::Unit::TestCase
         # Now define another type of comment, and make sure both types get
         # correctly returned as comments
         assert_nothing_raised do
-            parser.text_line :comment2, :match => /^"/
+            @parser.text_line :comment2, :match => /^"/
         end
 
         assert_nothing_raised("Did not parse old comment") do
             assert_equal({:record_type => :comment, :line => comment}, 
-                 parser.parse_line(comment))
+                 @parser.parse_line(comment))
         end
         comment = '" another type of comment'
         assert_nothing_raised("Did not parse new comment") do
             assert_equal({:record_type => :comment2, :line => comment}, 
-                 parser.parse_line(comment))
+                 @parser.parse_line(comment))
         end
 
         # Now define two overlapping record types and make sure we keep the
         # correct order.  We do first match, not longest match.
         assert_nothing_raised do
-            parser.text_line :one, :match => /^y/
-            parser.text_line :two, :match => /^yay/
+            @parser.text_line :one, :match => /^y/
+            @parser.text_line :two, :match => /^yay/
         end
 
         assert_nothing_raised do
             assert_equal({:record_type => :one, :line => "yayness"},
-                parser.parse_line("yayness"))
+                @parser.parse_line("yayness"))
         end
 
     end
 
     def test_record_line
-        parser = FParser.new
-
         tabrecord = "tab	separated	content"
         spacerecord = "space separated content"
 
         # Make sure we always require an appropriate set of options
         [{:separator => "\t"}, {}, {:fields => %w{record_type}}].each do |opts|
             assert_raise(ArgumentError, "Accepted %s" % opts.inspect) do
-                parser.record_line :record, opts
+                @parser.record_line :record, opts
             end
         end
 
         # Verify that our default separator is tabs
         tabs = nil
         assert_nothing_raised do
-            tabs = parser.record_line :tabs, :fields => [:name, :first, :second]
+            tabs = @parser.record_line :tabs, :fields => [:name, :first, :second]
         end
 
         # Make sure out tab line gets matched
         tabshould = {:record_type => :tabs, :name => "tab", :first => "separated",
                             :second => "content"}
         assert_nothing_raised do
-            assert_equal(tabshould, parser.handle_record_line(tabrecord, tabs))
+            assert_equal(tabshould, @parser.handle_record_line(tabrecord, tabs))
         end
 
         # Now add our space-separated record type
         spaces = nil
         assert_nothing_raised do
-            spaces = parser.record_line :spaces, :fields => [:name, :first, :second]
+            spaces = @parser.record_line :spaces, :fields => [:name, :first, :second]
         end
 
         # Now make sure both lines parse correctly
@@ -202,17 +199,15 @@ class TestUtilFileParsing < Test::Unit::TestCase
             :first => "separated", :second => "content"}
 
         assert_nothing_raised do
-            assert_equal(tabshould, parser.handle_record_line(tabrecord, tabs))
-            assert_equal(spaceshould, parser.handle_record_line(spacerecord, spaces))
+            assert_equal(tabshould, @parser.handle_record_line(tabrecord, tabs))
+            assert_equal(spaceshould, @parser.handle_record_line(spacerecord, spaces))
         end
     end
 
     def test_to_line
-        parser = FParser.new
-
-        parser.text_line :comment, :match => /^#/
-        parser.text_line :blank, :match => /^\s*$/
-        parser.record_line :record, :fields => %w{name one two}, :joiner => "\t"
+        @parser.text_line :comment, :match => /^#/
+        @parser.text_line :blank, :match => /^\s*$/
+        @parser.record_line :record, :fields => %w{name one two}, :joiner => "\t"
 
         johnny = {:record_type => :record, :name => "johnny", :one => "home",
             :two => "yay"}
@@ -236,7 +231,7 @@ class TestUtilFileParsing < Test::Unit::TestCase
         records.each do |name, details|
             result = nil
             assert_nothing_raised do
-                result = parser.to_line(details)
+                result = @parser.to_line(details)
             end
 
             assert_equal(lines[name], result)
@@ -248,55 +243,53 @@ class TestUtilFileParsing < Test::Unit::TestCase
         ordered_records = order.collect { |name| records[name] }
 
         # Make sure we default to a trailing separator
-        assert_equal(true, parser.trailing_separator,
+        assert_equal(true, @parser.trailing_separator,
             "Did not default to a trailing separtor")
 
         # Start without a trailing separator
-        parser.trailing_separator = false
+        @parser.trailing_separator = false
         assert_nothing_raised do
-            assert_equal(file, parser.to_file(ordered_records))
+            assert_equal(file, @parser.to_file(ordered_records))
         end
 
         # Now with a trailing separator
         file += "\n"
-        parser.trailing_separator = true
+        @parser.trailing_separator = true
         assert_nothing_raised do
-            assert_equal(file, parser.to_file(ordered_records))
+            assert_equal(file, @parser.to_file(ordered_records))
         end
 
         # Now try it with a different separator, so we're not just catching
         # defaults
         file.gsub!("\n", "\t")
-        parser.line_separator = "\t"
+        @parser.line_separator = "\t"
         assert_nothing_raised do
-            assert_equal(file, parser.to_file(ordered_records))
+            assert_equal(file, @parser.to_file(ordered_records))
         end
     end
 
     # Make sure fields that are marked absent get replaced with the appropriate
     # string.
     def test_absent_fields
-        parser = FParser.new
-
-        options = nil
+        record = nil
         assert_nothing_raised do
-            options = parser.record_line :record, :fields => %w{one two three},
+            record = @parser.record_line :record, :fields => %w{one two three},
                 :optional => %w{two three}
         end
-        assert_equal("", options[:absent], "Did not set a default absent string")
+        assert_equal("", record.absent, "Did not set a default absent string")
 
         result = nil
         assert_nothing_raised do
-            result = parser.to_line(:record_type => :record,
+            result = @parser.to_line(:record_type => :record,
                 :one => "a", :two => :absent, :three => "b")
         end
 
         assert_equal("a  b", result, "Absent was not correctly replaced")
 
         # Now try using a different replacement character
-        options[:absent] = "*" # Because cron is a pain in my ass
+        record.absent = "*" # Because cron is a pain in my ass
         assert_nothing_raised do
-            result = parser.to_line(:record_type => :record,
+            result = @parser.to_line(:record_type => :record,
                 :one => "a", :two => :absent, :three => "b")
         end
 
@@ -304,7 +297,7 @@ class TestUtilFileParsing < Test::Unit::TestCase
 
         # Make sure we deal correctly with the string 'absent'
         assert_nothing_raised do
-            result = parser.to_line(:record_type => :record,
+            result = @parser.to_line(:record_type => :record,
                 :one => "a", :two => "b", :three => 'absent')
         end
 
@@ -312,7 +305,7 @@ class TestUtilFileParsing < Test::Unit::TestCase
 
         # And, of course, make sure we can swap things around.
         assert_nothing_raised do
-            result = parser.to_line(:record_type => :record,
+            result = @parser.to_line(:record_type => :record,
                 :one => "a", :two => "b", :three => :absent)
         end
 
@@ -321,38 +314,36 @@ class TestUtilFileParsing < Test::Unit::TestCase
 
     # Make sure we can specify a different join character than split character
     def test_split_join_record_line
-        parser = FParser.new
-
         check = proc do |start, record, final|
             # Check parsing first
-            result = parser.parse_line(start)
+            result = @parser.parse_line(start)
             [:one, :two].each do |param|
                 assert_equal(record[param], result[param], 
                     "Did not correctly parse %s" % start.inspect)
             end
 
             # And generating
-            assert_equal(final, parser.to_line(result),
+            assert_equal(final, @parser.to_line(result),
                 "Did not correctly generate %s from %s" %
                 [final.inspect, record.inspect])
         end
 
         # First try it with symmetric characters
-        parser.record_line :symmetric, :fields => %w{one two},
+        @parser.record_line :symmetric, :fields => %w{one two},
             :separator => " "
 
         check.call "a b", {:one => "a", :two => "b"}, "a b"
-        parser.clear_records
+        @parser.clear_records
 
         # Now assymetric but both strings
-        parser.record_line :asymmetric, :fields => %w{one two},
+        @parser.record_line :asymmetric, :fields => %w{one two},
             :separator => "\t", :joiner => " "
 
         check.call "a\tb", {:one => "a", :two => "b"}, "a b"
-        parser.clear_records
+        @parser.clear_records
 
         # And assymmetric with a regex
-        parser.record_line :asymmetric2, :fields => %w{one two},
+        @parser.record_line :asymmetric2, :fields => %w{one two},
             :separator => /\s+/, :joiner => " "
 
         check.call "a\tb", {:one => "a", :two => "b"}, "a b"
@@ -361,11 +352,9 @@ class TestUtilFileParsing < Test::Unit::TestCase
 
     # Make sure we correctly regenerate files.
     def test_to_file
-        parser = FParser.new
-
-        parser.text_line :comment, :match => /^#/
-        parser.text_line :blank, :match => /^\s*$/
-        parser.record_line :record, :fields => %w{name one two}
+        @parser.text_line :comment, :match => /^#/
+        @parser.text_line :blank, :match => /^\s*$/
+        @parser.record_line :record, :fields => %w{name one two}
 
         text = "# This is a comment
 
@@ -374,33 +363,29 @@ billy three four\n"
 
         # Just parse and generate, to make sure it's isomorphic.
         assert_nothing_raised do
-            assert_equal(text, parser.to_file(parser.parse(text)),
+            assert_equal(text, @parser.to_file(@parser.parse(text)),
                 "parsing was not isomorphic")
         end
     end
 
     def test_valid_attrs
-        parser = FParser.new
-
-        parser.record_line :record, :fields => %w{one two three}
+        @parser.record_line :record, :fields => %w{one two three}
 
-        assert(parser.valid_attr?(:record, :one),
+        assert(@parser.valid_attr?(:record, :one),
             "one was considered invalid")
 
-        assert(parser.valid_attr?(:record, :ensure),
+        assert(@parser.valid_attr?(:record, :ensure),
             "ensure was considered invalid")
 
-        assert(! parser.valid_attr?(:record, :four),
+        assert(! @parser.valid_attr?(:record, :four),
             "four was considered valid")
     end
 
     def test_record_blocks
-        parser = FParser.new
-
         options = nil
         assert_nothing_raised do
             # Just do a simple test
-            options = parser.record_line :record,
+            options = @parser.record_line :record,
                 :fields => %w{name alias info} do |line|
                 line = line.dup
                 ret = {}
@@ -423,9 +408,6 @@ billy three four\n"
             end
         end
 
-        assert(parser.respond_to?(:handle_record_line_record),
-            "Parser did not define record method")
-
         values = {
             :name => "tcpmux",
             :description => "TCP port service multiplexer",
@@ -446,7 +428,7 @@ billy three four\n"
         }.each do |line, should|
             result = nil
             assert_nothing_raised do
-                result = parser.handle_record_line(line, options)
+                result = @parser.handle_record_line(line, options)
             end
             assert(result, "Did not get a result back for '%s'" % line)
             should.each do |field|
@@ -466,10 +448,8 @@ billy three four\n"
     # Make sure we correctly handle optional fields.  We'll skip this
     # functionality until we really know we need it.
     def test_optional_fields
-        parser = FParser.new
-
         assert_nothing_raised do
-            parser.record_line :record,
+            @parser.record_line :record,
                 :fields => %w{one two three four},
                 :optional => %w{three four},
                 :absent => "*",
@@ -479,13 +459,13 @@ billy three four\n"
         ["a b c d", "a b * d", "a b * *", "a b c *"].each do |line|
             record = nil
             assert_nothing_raised do
-                record = parser.parse_line(line)
+                record = @parser.parse_line(line)
             end
 
             # Now regenerate the line
             newline = nil
             assert_nothing_raised do
-                newline = parser.to_line(record)
+                newline = @parser.to_line(record)
             end
 
             # And make sure they're equal
@@ -494,41 +474,39 @@ billy three four\n"
 
         # Now make sure it pukes if we don't provide the required fields
         assert_raise(ArgumentError) do
-            parser.to_line(:record_type => :record, :one => "yay")
+            @parser.to_line(:record_type => :record, :one => "yay")
         end
     end
 
     def test_record_rts
-        parser = FParser.new
-
         # Start with the default
         assert_nothing_raised do
-            parser.record_line :record,
+            @parser.record_line :record,
                 :fields => %w{one two three four},
                 :optional => %w{three four}
         end
 
         assert_equal("a b  ",
-            parser.to_line(:record_type => :record, :one => "a", :two => "b")
+            @parser.to_line(:record_type => :record, :one => "a", :two => "b")
         )
 
         # Now say yes to removing
-        parser.clear_records
+        @parser.clear_records
         assert_nothing_raised do
-            parser.record_line :record,
+            @parser.record_line :record,
                 :fields => %w{one two three four},
                 :optional => %w{three four},
                 :rts => true
         end
 
         assert_equal("a b",
-            parser.to_line(:record_type => :record, :one => "a", :two => "b")
+            @parser.to_line(:record_type => :record, :one => "a", :two => "b")
         )
 
         # Lastly, try a regex
-        parser.clear_records
+        @parser.clear_records
         assert_nothing_raised do
-            parser.record_line :record,
+            @parser.record_line :record,
                 :fields => %w{one two three four},
                 :optional => %w{three four},
                 :absent => "*",
@@ -536,9 +514,138 @@ billy three four\n"
         end
 
         assert_equal("a b",
-            parser.to_line(:record_type => :record, :one => "a", :two => "b")
+            @parser.to_line(:record_type => :record, :one => "a", :two => "b")
         )
     end
+
+    # Make sure the last field can contain the separator, as crontabs do, and
+    # that we roll them all up by default.
+    def test_field_rollups
+        @parser.record_line :yes, :fields => %w{name one two}
+
+        result = nil
+        assert_nothing_raised do
+            result = @parser.send(:parse_line, "Name One Two Three")
+        end
+        assert_equal("Two Three", result[:two],
+            "Did not roll up last fields by default")
+
+        @parser = FParser.new
+        assert_nothing_raised("Could not create record that rolls up fields") do
+            @parser.record_line :no, :fields => %w{name one two}, :rollup => false
+        end
+
+        result = nil
+        assert_nothing_raised do
+            result = @parser.send(:parse_line, "Name One Two Three")
+        end
+        assert_equal("Two", result[:two],
+            "Rolled up last fields when rollup => false")
+    end
+
+    def test_text_blocks
+        record = nil
+        assert_nothing_raised do
+            record = @parser.text_line :name, :match => %r{^#} do |line|
+                {:line => line.upcase}
+            end
+        end
+
+        assert(record.respond_to?(:process),
+            "Block was not used with text line")
+
+        assert_equal("YAYNESS", record.process("yayness")[:line],
+            "Did not call process method")
+    end
+
+    def test_hooks
+        record = nil
+        # First try it with a normal record
+        assert_nothing_raised("Could not set hooks") do
+            record = @parser.record_line :yay, :fields => %w{one two},
+                :post_parse => proc { |hash| hash[:posted] = true },
+                :pre_gen => proc { |hash| hash[:one] = hash[:one].upcase }
+        end
+
+        assert(record.respond_to?(:post_parse), "did not create method for post-hook")
+        assert(record.respond_to?(:pre_gen), "did not create method for pre-hook")
+
+        result = nil
+        assert_nothing_raised("Could not process line with hooks") do
+            result = @parser.parse_line("one two")
+        end
+
+        assert(result[:posted], "Did not run post-hook")
+        old_result = result
+
+        # Now make sure our pre-gen hook is called
+        assert_nothing_raised("Could not generate line with hooks") do
+            result = @parser.to_line(result)
+        end
+        assert_equal("ONE two", result, "did not call pre-gen hook")
+        assert_equal("one", old_result[:one], "passed original hash to pre hook")
+    end
+end
+
+class TestUtilFileRecord < Test::Unit::TestCase
+	include PuppetTest
+	include PuppetTest::FileParsing
+
+    Record = Puppet::Util::FileParsing::FileRecord
+    def test_new_filerecord
+        [   [:fake, {}],
+            [nil, ]
+        ].each do |args|
+            assert_raise(ArgumentError, "Did not fail on %s" % args.inspect) do
+                Record.new(*args)
+            end
+        end
+
+        # Make sure the fields get turned into symbols
+        record = nil
+        assert_nothing_raised do
+            record = Record.new(:record, :fields => %w{one two})
+        end
+        assert_equal([:one, :two], record.fields,
+            "Did not symbolize fields")
+
+        # Make sure we fail on invalid fields
+        [:record_type, :target, :on_disk].each do |field|
+            assert_raise(ArgumentError, "Did not fail on invalid field %s" % field) {
+                Record.new(:record, :fields => [field])
+            }
+        end
+    end
+
+    def test_defaults
+        record = Record.new(:text, :match => %r{^#})
+        [:absent, :separator, :joiner, :optional].each do |field|
+            assert_nil(record.send(field), "%s was not nil" % field)
+        end
+
+        record = Record.new(:record, :fields => %w{fields})
+        {:absent => "", :separator => /\s+/, :joiner => " ",
+            :optional => []}.each do |field, default|
+                assert_equal(default, record.send(field), "%s was not default" % field)
+        end
+    end
+
+    def test_block
+        record = nil
+        assert_nothing_raised("Could not pass a block when creating record") do
+            record = Record.new(:record, :fields => %w{one}) do |line|
+                return line.upcase
+            end
+        end
+
+        line = "This is a line"
+
+        assert(record.respond_to?(:process),
+            "Record did not define :process method")
+
+        assert_equal(line.upcase, record.process(line),
+            "Record did not process line correctly")
+    end
 end
 
 # $Id$