Updates to everest-bootstrap to support everestd

These changes are to support a oneliner provisioning story in the Everest
environment.

New features:
* We use facts instead of parameters.  The can be create prior to koaning a
machine.  They can also be managed centrally in a more efficient manner.
* uses the cobbler xmlrpc api to allow users to select which OS that would like
to use for their machine type.
* it's now possible to optionally use Red Hat's DDNS
* added a '--config-only' option.  This does not need to be run as root in that
case. This is useful if you simpy want to update a machine's parameters.  This
will also be useful when it comes time to port machines over to the use of
parameters. The machines already exist therefore they don't need to be koaned.

TODO:
everest-bootstrap will kick of koan but after it finishes the machine will not
start back up.  We can either do some trickery to make the tool smart enough to
know when it finishes or we can try and find a way for Xen to reboot the
machine automatically.

4 files changed, 147 insertions, 164 deletions

diff --git a/everest-bootstrap/Rakefile b/everest-bootstrap/Rakefile
index 5a3c830..8574286 100644
--- a/everest-bootstrap/Rakefile
+++ b/everest-bootstrap/Rakefile
@@ -12,7 +12,7 @@ Hoe.new('everest-bootstrap', EverestBootstrap::Version::STRING) do |p|
   p.description = 'Tool for provisioning virtual machines'
   # p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
   p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
-  p.extra_deps = %w[highline main everest]
+  p.extra_deps = %w[highline main reststop]
 end
 
 # vim: syntax=Ruby
diff --git a/everest-bootstrap/bin/everest-bootstrap b/everest-bootstrap/bin/everest-bootstrap
index a5c93d9..0f4ac50 100644
--- a/everest-bootstrap/bin/everest-bootstrap
+++ b/everest-bootstrap/bin/everest-bootstrap
@@ -18,9 +18,15 @@ Main {
   end unless $DEBUG
 
   def get_input
-    @user = ask("Enter your kerberos username: ")
-    @password = ask("Enter your kerberos password to setup DDNS: ") { |q| q.echo = "*" }
+    unless ENV['USER'] == 'root' || params["config-only"].given?
+      raise "You must run this command as root unless " +
+            "you specify the '--config-only' option" 
+    end
+
+    @facts = Hash.new
+
     @repo = ask("Enter your Everest repo machine name (leave off the '-repo'): ")
+    @user = ask("Enter your kerberos username: ")
 
     say("Your machine name is the part right before the Everest type.")
     say("For example, 'sso1-build.usersys.redhat.com' has a machine name of 'sso1'.")
@@ -32,22 +38,43 @@ Main {
       @machine_name = @user
     end
 
+    # Once we know the repo we can fetch the supported machines
+    @everest_repo = EverestRepo.new(@repo)
     say("To see a description of these machine types visit: " +
-        "http://#{@repo}-repo-usersys.redhat.com/cgi-bin/machine_types.cgi")
-    # Once we know the repo and machine name we can fetch the supported machines
-    @fetcher = MachineFetcher.new(@repo, @machine_name)
+        "#{@everest_repo.machine_types_url}")
 
     @machine_type = choose do |menu|
       menu.prompt = "Select your machine type:  "
-      menu.choices(*@fetcher.machines.map{|m| m.name})
+      menu.choices(*@everest_repo.machines.map{|m| m.name})
     end
 
-    say("Now you need to enter facts about your machine.  These facts will be " +
-      "used by puppet to configure the system")
-    say("You will notice most of the defaults will work out just fine")
+    # This might be useful is we stop using DDNS
+    @facts["everest_machine_type"] = @machine_type
 
-    @facts = Hash.new
-    @fetcher.facts_for(@machine_type).each do |f|
+    if agree("Configure this machine with Red Hat DDNS? (y/n)", true)
+      # Our naming convention
+      hostname = @machine_name + "-" + @machine_type
+      @fqdn = hostname + ".usersys.redhat.com"
+
+      if agree("Do you know the DDNS hash for this machine? (y/n)", true)
+        @facts["rh_ddns_hash"] = ask("Enter Red Hat DDNS hash: ") 
+      else 
+        @password = ask("Enter your kerberos password to setup DDNS: ") { |q| q.echo = "*" }
+        @facts["rh_ddns_hash"] = RedHatDDNS::DDNS.new(@user, @password, hostname).ddns_hash
+      end
+    else 
+      @fqdn = ask("Enter fully qualified domain name for this machine: ")
+    end
+
+    @facts["cobbler_profile"] = choose do |menu|
+      menu.prompt = "Select your cobbler profile (This determines the OS):  "
+      menu.choices(*@everest_repo.cobbler_profiles)
+    end
+
+    say("Now you need to enter some parameters what will be used to configure this machine.")
+    say("Most of the defaults will work out fine for development.")
+
+    @everest_repo.facts_for(@machine_type, @machine_name).each do |f|
       say("Description: #{f.desc}") unless f.desc.empty?
       @facts[f.name] = ask("Enter value for #{f.name}: ") do |q|
         q.default = f.default
@@ -62,25 +89,34 @@ Main {
       end while agree("Enter another? ", true)
     end
 
-    if @guest = agree("Are you bootstrapping a guest? (y/n)", true)
-      say("The virt path is the path to your virtual machine.  If backed by LVM they " +
-          "can usually be found under /dev/<Your Volume Group>")
+    unless params["config-only"].given?
+      if agree("Use LVM for VM storage (recommended)? (y/n)", true)
+        @virt_path = choose do |menu|
+          menu.prompt = "Select the Volume Group to use: "
+          menu.choices(*(`/usr/sbin/vgdisplay --short`.split("\n").map {|l| /"(.*)"/.match(l).captures[0]}))
+        end
+      else 
+        @virt_path = ask("Enter the flat file to be used for guest: ") 
+      end
 
-      # Highline's readline support doesn't complete paths.  That's lame.
-      require 'readline'
-      # readline captures the whole line, that's why we have to strip
-      @virt_path = Readline::readline('Enter The virt-path: ').strip
-      puts File.exist?(@virt_path)
+      say("The Koan process is now starting.  After it finishes start the VM back up with " + 
+          "'xm create <name>'.")
+      say("HINT: 'xm list' will show you the running VMs.")
     end
   end
 
+  option('config-only', 'c'){
+    description "Only send the machine configuration to the Puppetmaster"
+  }
+
   def run
     get_input
-    EverestMachine.new(@machine_type,
-                       @machine_name,
-                       @user,
-                       @password,
-                       @repo,
-                       @facts).bootstrap(@guest, @virt_path)
+    e = EverestMachine.new(@machine_type,
+                           @fqdn,
+                           @repo,
+                           @facts)
+    e.post_config 
+    e.add_system_to_cobbler
+    e.koan(@virt_path) unless params["config-only"].given?   
   end
 }
diff --git a/everest-bootstrap/lib/everest-bootstrap/core.rb b/everest-bootstrap/lib/everest-bootstrap/core.rb
index 0b4a39d..2bae4f4 100644
--- a/everest-bootstrap/lib/everest-bootstrap/core.rb
+++ b/everest-bootstrap/lib/everest-bootstrap/core.rb
@@ -1,173 +1,120 @@
 require 'erb'
 require 'yaml'
-require 'everest'
+require 'restr'
+require 'xmlrpc/client'
 
 module EverestBootstrap
-  class MachineFetcher
-    attr_reader :machines
-
-    # I don't like giving this class two responsibilities
-    def initialize(repo, machine_name="") 
-      @machine_name = machine_name
-      @repo = repo
-      @machines = fetch_data(@repo)
-    end
+  class EverestRepo
+    attr_reader :machines, :machine_types_url, :cobbler_profiles, :fqdn
 
-    def machine_types
-      @machines.map {|m| m.name.to_s}
-    end
+    def initialize(repo_name) 
+      @repo_name = repo_name
+      @fqdn = "#{@repo_name}-repo.usersys.redhat.com"
+      @everestd = "http://#{@fqdn}:8106"
+      @machine_types_url = @everestd + "/machine_types.html"
 
-    def replace_values(text)
-      text.gsub("%repo%", @repo).gsub("%machine_name%", @machine_name)
+      @cobblerd = XMLRPC::Client.new2("http://#{@fqdn}:25152")           
+
+      # TODO: figure out a way to pass these values in
+      @token = @cobblerd.call2("login", "cobbler", "password")[1]
+      @cobbler_profiles = @cobblerd.call2("get_profiles", @token)[1].map {|p| p["name"]}
     end
 
-    def facts_for(name)
-      facts = @machines.find {|m| m.name == name}.facts
-      facts.each {|f| f.set_default(replace_values(f.default))}
-      return facts
+    def machines
+      @machines ||= fetch_data
     end
 
-    # Fetch the object returned and load it back.  The classes for this object
-    # are in the everest gem
-    def fetch_data(repo)
-      url = "http://#{repo}-repo.usersys.redhat.com/cgi-bin/machine_dump.cgi"
-
-      # We should really catch some exceptions here
-      Net::HTTP.get_response(URI.parse(url)) do |res|
-        if res.code == "200" 
-          return Marshal.load(res.body)
-        else 
-          raise "#{url} returned a bad response"
+    def facts_for(type, machine_name)
+      # Don't want to pull in the everest lib.  See below fore more detail.
+      facts = Restr.get("#{@everestd}/machine_types/#{type}.xml")["fact"]
+      return facts.map do |f|
+        def f.name
+          self["name"]
         end
-      end
-    end
-  end
 
-  module Bootstrappable
-    COMMANDS = []
-    def command(name, &task)
-      COMMANDS << OpenStruct.new(:name => name, :task => task)
-    end
+        def f.desc
+          self["description"]  
+        end
 
-    def sh(cmd)
-      puts cmd
-      print `#{cmd}`
-    end
-    
-    # We want people to be able to type a fully qualified path
-    # or just the logical volume name
-    def determine_virt_path(vpath)
-      # If the path doesn't start with a '/', assume it's a logical volume
-      unless /^\// =~ vpath
-        vpath = build_lvm_path
+        # Here's where we make use of the DSL's simple templating
+        f.instance_variable_set(:@repo_name, @repo_name)
+        f.instance_variable_set(:@machine_name, machine_name)
+        def f.default
+          self["default"].gsub("%repo%", @repo_name).gsub("%machine_name%", @machine_name)
+        end
+        
+        f
       end
-
-      return vpath
     end
 
-    def build_lvm_path
-      # Find the volume group for the specified LVM name
-      # Get a table of output with the lvm name and volume group
-      lvs_table = `/usr/sbin/lvs --noheadings -o lv_name,vg_name`.split
-
-      # Find the matching logical volume and read corresponding volume group
-      volume_group = lvs_table[lvs_table.index(params['virt-path'].value) + 1]
-
-      # Build up the device path
-      return "/dev/#{volume_group}/#{params['virt-path'].value}"
+    def classes_for(type)
+      Restr.get("#{@everestd}/machine_types/#{type}.xml")["class"]
     end
 
-    def bootstrap_vm(path)
-      virt_path = determine_virt_path(path)
-      
-      puts "Mounting the guests logical volume..." 
-      sh "mkdir /mnt/#{hostname}"
-
-      if File.blockdev? virt_path
-        sh "/sbin/kpartx -a #{virt_path}"
-        sh "mount /dev/mapper/#{File.basename(virt_path)}p1 /mnt/#{hostname}"
-      else
-        # This is not a block device, so we have to access it through a loopback
-        sh "/sbin/losetup /dev/loop0 #{virt_path}"
-        sh "/sbin/kpartx -a /dev/loop0"
-        sh "mount /dev/mapper/loop0p1 /mnt/#{hostname}"
-      end
-
-      write_firstboot_script(:vm => true)
-
-      puts "Cleaning up..."
-      sh "umount -f /mnt/#{hostname}"
+    def fetch_data
+      # Build out some mock objects so we don't have to couple to the everest
+      # lib.  That lib has lots of funky metaprogramming stuff in it and it
+      # might need to be changed in the future.  The fewer tools relying on it
+      # the better.
+      machine_types = Restr.get("#{@everestd}/machine_types.xml")["machine_type"]
+      return machine_types.map do |m|
+        def m.name
+          self["name"]
+        end
 
-      if File.blockdev? virt_path
-        sh "/sbin/kpartx -d #{virt_path}"
-      else
-        # This is not a block device, so cleanup the loopback
-        sh "/sbin/kpartx -d /dev/loop0"
-        sh "/sbin/losetup -d /dev/loop0"
+        def m.desc
+          self["description"]  
+        end
+         
+        m
       end
-
-      sh "rmdir /mnt/#{hostname}"
-
-      puts "Finished configuring puppet to run on first boot."
-      puts "The next time you start this machine, puppet will configure it as #{fqdn}"
-      puts "Once you are logged in `tail -f /var/log/messages` to see how puppet is doing."
-    end
-
-    def bootstrap_self
-      write_firstboot_script
     end
-
-    def write_firstboot_script(options={:vm => false})
-      firstboot_path = "/usr/sbin/everest-firstboot"
-      sysconfig_path = "/etc/sysconfig/everest-firstboot"
-
-      # If this is a vm, use the vm mount point
-      firstboot_path = "/mnt/#{hostname}" + firstboot_path if options[:vm]
-      sysconfig_path = "/mnt/#{hostname}" + sysconfig_path if options[:vm]
-
-      # This firstboot script is executed by a firstboot-like service script
-      # that is layed down at provisioning time by an rpm called everest-firstboot.
-      # That service wrapper in that rpm is configured to run at system startup
-      # which checks the /etc/sysconfig file to determine whether or not to run
-      # the dynamically generated puppet firstboot file.
-      puts "Laying down firstboot script"
-      File.open(firstboot_path, "w") do |f|
-        template = File.read(File.dirname(__FILE__) + "/everest-firstboot.erb")
-        f.puts ERB.new(template).result(binding)
-      end
-
-      File.chmod(0755, firstboot_path)
-      File.open(sysconfig_path, "w") { |f| f.puts "RUN_BOOTSTRAP=YES" }
+ 
+    def post_config(node_name, machine_type, params)
+      classes = classes_for(machine_type)
+      config = YAML.dump({"classes" => classes, "parameters" => params})
+      Restr.post("#{@everestd}/nodes", :node_name => node_name, :yaml => config)
     end
 
-    # This is kind of silly, but it's late and my wife is pregnant.
-    def bootstrap(vm, virt_path) 
-      if vm
-        bootstrap_vm(virt_path)
-      else
-        bootstrap_self
-      end
+    def add_system_to_cobbler(name, params)
+      system_id = @cobblerd.call2("new_system", @token)[1]
+      @cobblerd.call2('modify_system', system_id, 'name', name, @token)
+      @cobblerd.call2('modify_system', system_id, 'profile', params["cobbler_profile"], @token)
+   
+      # These values ultimately get used by everest-firstboot
+      ksmeta = "fqdn=#{name} " +
+               "everest_repo=#{@fqdn}"
+
+      @cobblerd.call2('modify_system', system_id, 'ksmeta', ksmeta, @token)
+      @cobblerd.call2('save_system', system_id, @token)
     end
   end
 
   class EverestMachine
     include RedHatDDNS
-    include Bootstrappable
     attr_reader :type, :hostname, :fqdn
 
-    def initialize(machine_type, machine_name, kerb_user, password, repo, facts)
+    def initialize(machine_type, fqdn, repo_name, facts)
       @machine_type = machine_type
-      @machine_name = machine_name
-      @kerb_user = kerb_user
-      @password = password
-      @hostname = @machine_name + "-" + @machine_type
-      @fqdn = @hostname + ".usersys.redhat.com"
-      @repo = repo + "-repo"
+      @fqdn = fqdn
+      @repo_name = repo_name + "-repo"
       @facts = facts
+      @everest_repo = EverestRepo.new(repo_name)
     end
 
-    def ddns_hash
-      @ddns_hash ||= DDNS.new(@kerb_user, @password, @hostname)
+    def post_config
+      @everest_repo.post_config(@fqdn, @machine_type, @facts)
+    end
+
+    def add_system_to_cobbler
+      @everest_repo.add_system_to_cobbler(@fqdn, @facts)
+    end
+ 
+    def koan(virt_path)
+      koan_command = "/usr/bin/koan -s #{@everest_repo.fqdn} " +
+                     "--virt --virt-type=xenpv --virt-path=#{virt_path} " +
+                     "--system=#{@fqdn} --virt-name=#{@fqdn.split(".")[0]}"
+      `#{koan_command}` 
     end
   end
 end
diff --git a/everest-bootstrap/lib/everest-bootstrap/version.rb b/everest-bootstrap/lib/everest-bootstrap/version.rb
index 5874235..92d5d9d 100644
--- a/everest-bootstrap/lib/everest-bootstrap/version.rb
+++ b/everest-bootstrap/lib/everest-bootstrap/version.rb
@@ -1,8 +1,8 @@
 module EverestBootstrap
   module Version
     MAJOR = 0
-    MINOR = 3
-    BUILD = 2
+    MINOR = 4
+    BUILD = 0
 
     STRING = [MAJOR, MINOR, BUILD].join(".")
   end