A simple prototype for backing up Xen guests to S3

13 files changed, 458 insertions, 0 deletions

diff --git a/everest-bootstrap/.gitignore b/everest-bootstrap/.gitignore
new file mode 100644
index 0000000..3e57bca
--- /dev/null
+++ b/everest-bootstrap/.gitignore
@@ -0,0 +1,3 @@
+*.swp
+*.swo
+pkg/*
diff --git a/everest-bootstrap/History.txt b/everest-bootstrap/History.txt
new file mode 100644
index 0000000..7bb5120
--- /dev/null
+++ b/everest-bootstrap/History.txt
@@ -0,0 +1,2 @@
+== 0.1.0 / 2008-01-18
+  * Birthday!
diff --git a/everest-bootstrap/Manifest.txt b/everest-bootstrap/Manifest.txt
new file mode 100644
index 0000000..d61736c
--- /dev/null
+++ b/everest-bootstrap/Manifest.txt
@@ -0,0 +1,12 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/everest-bootstrap
+lib/everest-bootstrap.rb
+lib/everest-bootstrap/core.rb
+lib/everest-bootstrap/commands.rb
+lib/everest-bootstrap/ddns.rb
+lib/everest-bootstrap/version.rb
+ext/rubygem-everest-bootstrap.spec
+test/test_everest-bootstrap.rb
diff --git a/everest-bootstrap/README.txt b/everest-bootstrap/README.txt
new file mode 100644
index 0000000..c079afb
--- /dev/null
+++ b/everest-bootstrap/README.txt
@@ -0,0 +1,43 @@
+everest-bootstrap
+    by it-arch 
+
+== DESCRIPTION:
+  
+This is a provisioning too for IT puppet machines.
+
+== FEATURES/PROBLEMS:
+  
+* CLI interface 
+
+== SYNOPSIS:
+
+  everest-bootstrap -h 
+
+== INSTALL:
+
+* sudo gem/yum install everest-bootstrap
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007 FIX
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/everest-bootstrap/Rakefile b/everest-bootstrap/Rakefile
new file mode 100644
index 0000000..4e0517d
--- /dev/null
+++ b/everest-bootstrap/Rakefile
@@ -0,0 +1,18 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './lib/everest-bootstrap/version.rb'
+
+Hoe.new('everest-bootstrap', EverestBootstrap::Version::STRING) do |p|
+  p.author = 'it-arch'
+  p.email = "gis-java@redhat.com"
+  p.rubyforge_name = 'everest-bootstrap'
+  p.summary = 'Tool for provisioning virtual machines'
+  p.description = 'Tool for provisioning virtual machines'
+  # p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.extra_deps = %w[highline main]
+end
+
+# vim: syntax=Ruby
diff --git a/everest-bootstrap/bin/everest-bootstrap b/everest-bootstrap/bin/everest-bootstrap
new file mode 100644
index 0000000..855d065
--- /dev/null
+++ b/everest-bootstrap/bin/everest-bootstrap
@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+require 'ostruct'
+require 'everest-bootstrap'
+require 'main'
+require 'net/http'
+require 'uri'
+
+COMMANDS = []
+def command(name, &task)
+  COMMANDS << OpenStruct.new(:name => name, :task => task)
+end
+
+def sh(cmd)
+  puts cmd
+  print `#{cmd}`
+end
+
+# Load the commands
+instance_eval(File.read(File.dirname(__FILE__) + "/../lib/everest-bootstrap/commands.rb"))
+
+Main {
+  version EverestBootstrap::Version::STRING
+
+  def handle_exception e
+    puts e.message
+  end unless $DEBUG
+
+  def get_node_types(machine_name="everest")
+    url = "http://#{machine_name}-repo.usersys.redhat.com/cgi-bin/nodetypes.cgi"
+    Net::HTTP.get_response(URI.parse(url)) do |res|
+      if res.code == "200" 
+        return res.body.split(" ").sort 
+      else 
+        error "#{url} returned a bad response"
+        return Array.new
+      end
+    end
+  end
+
+  mode 'setup' do
+    option("machine-type") do
+      required
+      argument_required
+      description "Run 'everest-boostrap list-machines to see valid types"
+    end
+
+    option("machine-name") do
+      argument_required
+      description 'The name of the machine (the part of the hostname before the machine type)'
+    end
+
+    option("repo", "r") do
+      argument_required
+      description "Repo machine to use for puppet/git/archiva (leave off the '-repo' part)"
+      default 'everest'
+    end
+
+    option("user", "u") do
+      required
+      argument_required
+      description 'Your kerberos username'
+    end
+
+    option("facts", "f") do
+      argument_required
+      description 'Additional facts to lay down on the guest.  Format is --facts=factname=foo|factname2=bar'
+    end
+
+    mode 'guest' do
+      def everest_machine
+        @everest_machine ||= EverestBootstrap::EverestMachine.new(params['machine-type'].value,
+                                                                  params['machine-name'].given? ? params['machine-name'].value : params['user'].value,
+                                                                  params['user'].value,
+                                                                  params['repo'].value,
+                                                                  params['facts'].value)
+      end
+
+      option("virt-path", "p") do
+        required
+        argument_required
+        description 'The path to the virtual image (this could be a file path or a logical volume name)'
+      end
+
+      COMMANDS.each do |c|
+        mode c.name do
+          # We are forced to dynamically define the run method
+          # for the modules.  Otherwise our 'c' would be out of
+          # scope.
+          define_method(:run) {instance_eval(&c.task)}
+        end
+      end
+
+      # Figure out the virt-path destination
+      def virt_path
+        vpath = params['virt-path'].value
+
+        # If the path doesn't start with a '/', assume it's a logical volume
+        unless /^\// =~ vpath
+          vpath = build_lvm_path
+        end
+
+        return vpath
+      end
+
+      def build_lvm_path
+        # Find the volume group for the specified LVM name
+        # Get a table of output with the lvm name and volume group
+        lvs_table = `/usr/sbin/lvs --noheadings -o lv_name,vg_name`.split
+
+        # Find the matching logical volume and read corresponding volume group
+        volume_group = lvs_table[lvs_table.index(params['virt-path'].value) + 1]
+
+        # Build up the device path
+        return "/dev/#{volume_group}/#{params['virt-path'].value}"
+      end
+
+      # Run all the commands in order
+      def run
+        COMMANDS.each {|c| instance_eval(&c.task)}
+      end
+    end
+
+    mode 'host' do
+      def everest_machine
+        @everest_machine ||= EverestBootstrap::EverestMachine.new(params['machine-type'].value,
+                                                                params['machine-name'].given? ? params['machine-name'].value : params['user'].value,
+                                                                params['user'].value,
+                                                                params['repo'].value,
+                                                                params['facts'].value,
+                                                                :vm => false)
+      end
+
+      def run
+        puts "Laying down firstboot script"
+        everest_machine.bootstrap
+      end
+    end
+  end 
+ 
+  mode 'list-machines' do
+    option("repo", "r") do
+        argument_required
+        description "Repo machine to use for puppet/git/artifactory (leave off the '-repo' part)"
+        default 'everest'
+    end
+
+    def run() 
+        repo = params["repo"].given? ? params["repo"].value : "everest"
+        puts get_node_types(repo)
+    end
+  end
+}
diff --git a/everest-bootstrap/ext/rubygem-everest-bootstrap.spec b/everest-bootstrap/ext/rubygem-everest-bootstrap.spec
new file mode 100644
index 0000000..52f54cc
--- /dev/null
+++ b/everest-bootstrap/ext/rubygem-everest-bootstrap.spec
@@ -0,0 +1,60 @@
+%define ruby_sitelib %(ruby -rrbconfig -e "puts Config::CONFIG['sitelibdir']")
+%define gemdir %(ruby -rubygems -e 'puts Gem::dir' 2>/dev/null)
+%define gemname everest-bootstrap
+%define geminstdir %{gemdir}/gems/%{gemname}-%{version}
+
+Summary: Tool for provisioning virtual machines
+Name: rubygem-%{gemname}
+
+Version: 0.2.2
+Release: 1%{?dist}
+Group: Development/Languages
+License: Ruby License/GPL
+Source0: %{gemname}-%{version}.gem
+BuildRoot: %{_tmppath}/%{name}-%{version}-root-%(%{__id_u} -n)
+Requires: rubygems
+Requires: rubygem(highline) 
+Requires: rubygem(main) 
+Requires: rubygem(hoe) >= 1.3.0
+Requires: wget
+Requires: lvm2
+BuildRequires: rubygems
+BuildArch: noarch
+Provides: rubygem(%{gemname}) = %{version}
+Obsoletes: everest-bootstrap
+Obsoletes: rake
+Obsoletes: hoe
+Obsoletes: highline
+Obsoletes: rubyforge
+Obsoletes: rvm
+
+%description
+Tool for bootstrapping the puppet configuration on machine images
+
+
+%prep
+
+%build
+
+%install
+%{__rm} -rf %{buildroot}
+mkdir -p %{buildroot}%{gemdir}
+gem install --local --install-dir %{buildroot}%{gemdir} --force --rdoc %{SOURCE0}
+mkdir -p %{buildroot}/%{_bindir}
+mv %{buildroot}%{gemdir}/bin/* %{buildroot}/%{_bindir}
+rmdir %{buildroot}%{gemdir}/bin
+find %{buildroot}%{geminstdir}/bin -type f | xargs chmod a+x
+
+%clean
+%{__rm} -rf %{buildroot}
+
+%files
+%defattr(-, root, root)
+%{_bindir}/everest-bootstrap
+%{gemdir}/gems/%{gemname}-%{version}/
+%doc %{gemdir}/doc/%{gemname}-%{version}
+%doc %{geminstdir}/History.txt
+%doc %{geminstdir}/Manifest.txt
+%doc %{geminstdir}/README.txt
+%{gemdir}/cache/%{gemname}-%{version}.gem
+%{gemdir}/specifications/%{gemname}-%{version}.gemspec
diff --git a/everest-bootstrap/lib/everest-bootstrap.rb b/everest-bootstrap/lib/everest-bootstrap.rb
new file mode 100644
index 0000000..be3f402
--- /dev/null
+++ b/everest-bootstrap/lib/everest-bootstrap.rb
@@ -0,0 +1,7 @@
+require 'optparse'
+require 'ostruct'
+require 'open3'
+require 'everest-bootstrap/ddns'
+require 'everest-bootstrap/core'
+require 'everest-bootstrap/version'
+require 'highline'
diff --git a/everest-bootstrap/lib/everest-bootstrap/commands.rb b/everest-bootstrap/lib/everest-bootstrap/commands.rb
new file mode 100644
index 0000000..9e0f9f8
--- /dev/null
+++ b/everest-bootstrap/lib/everest-bootstrap/commands.rb
@@ -0,0 +1,41 @@
+# These commands will be executed in the order they are defined
+
+command :mount do
+  puts "Mounting the guests logical volume..." 
+  sh "mkdir /mnt/#{everest_machine.fqmn}"
+
+  if File.blockdev? virt_path
+    sh "/sbin/kpartx -a #{virt_path}"
+    sh "mount /dev/mapper/#{File.basename(virt_path)}p1 /mnt/#{everest_machine.fqmn}"
+  else
+    # This is not a block device, so we have to access it through a loopback
+    sh "/sbin/losetup /dev/loop0 #{virt_path}"
+    sh "/sbin/kpartx -a /dev/loop0"
+    sh "mount /dev/mapper/loop0p1 /mnt/#{everest_machine.fqmn}"
+  end
+end
+
+command :prime_firstboot do
+  everest_machine.bootstrap
+end
+
+command :unmount do
+  puts "Cleaning up..."
+  sh "umount /mnt/#{everest_machine.fqmn}"
+
+  if File.blockdev? virt_path
+    sh "/sbin/kpartx -d #{virt_path}"
+  else
+    # This is not a block device, so cleanup the loopback
+    sh "/sbin/kpartx -d /dev/loop0"
+    sh "/sbin/losetup -d /dev/loop0"
+  end
+
+  sh "rmdir /mnt/#{everest_machine.fqmn}"
+end
+
+command :finish do
+  puts "Finished configuring puppet to run on first boot."
+  puts "The next time you start this machine, puppet will configure it as #{everest_machine.fqdn}"
+  puts "Once you are logged in `tail -f /var/log/messages` to see how puppet is doing."
+end
diff --git a/everest-bootstrap/lib/everest-bootstrap/core.rb b/everest-bootstrap/lib/everest-bootstrap/core.rb
new file mode 100644
index 0000000..377795e
--- /dev/null
+++ b/everest-bootstrap/lib/everest-bootstrap/core.rb
@@ -0,0 +1,54 @@
+module EverestBootstrap
+  class EverestMachine
+    include RedHatDDNS
+    attr_reader :type, :hostname, :fqdn, :fqmn
+
+    def initialize(machine_type, machine_name, kerb_user, repo, facts, options = {:vm => true})
+      @kerb_user = kerb_user
+      @machine_name = machine_name
+      # Fully qualified machine name (like 'team1Repo' or 'bleanharBuild')
+      @fqmn = machine_name + machine_type
+      # This is the "Everest" machine type.  ie, web-build instead of WebBuild
+      @hostname_suffix = machine_type.gsub(/(.)([A-Z])/, '\1-\2').downcase
+      @hostname = @machine_name + "-" + @hostname_suffix
+      @fqdn = @hostname + ".usersys.redhat.com"
+      @repo = repo + "-repo"
+      @ui = HighLine.new
+      @password = @ui.ask("Enter your kerberos password to setup DDNS: ") { |q| q.echo = "*" }
+      @facts = facts
+      @vm = options[:vm]
+    end
+
+    def ddns_hash
+      @ddns_hash ||= DDNS.new(@kerb_user, @password, @hostname)
+    end
+
+    def bootstrap
+      firstboot_path = "/usr/sbin/everest-firstboot"
+      sysconfig_path = "/etc/sysconfig/everest-firstboot"
+
+      # If this is a vm, use the vm mount point
+      firstboot_path = "/mnt/#{@fqmn}" + firstboot_path if @vm
+      sysconfig_path = "/mnt/#{@fqmn}" + sysconfig_path if @vm
+
+      # This firstboot script is executed by a firstboot-like service script
+      # that is layed down at provisioning time by an rpm called everest-firstboot.
+      # That service wrapper in that rpm is configured to run at system startup
+      # which checks the /etc/sysconfig file to determine whether or not to run
+      # the dynamically generated puppet firstboot file.
+      puts "Laying down firstboot script"
+      File.open(firstboot_path, "w") do |f|
+        f.puts `#{"curl --get -d template=bootstrap " +
+                 "-d node_type=#{@hostname_suffix} " +
+                 "-d username=#{@kerb_user} " +
+                 "-d machine_name=#{@machine_name} " + 
+                 "-d ddns_hash=#{ddns_hash} " + 
+                 "-d facts=#{@facts} " +
+                 "http://#{@repo}.usersys.redhat.com/cgi-bin/everest.cgi"}`
+      end
+
+      File.chmod(0755, firstboot_path)
+      File.open(sysconfig_path, "w") { |f| f.puts "RUN_BOOTSTRAP=YES" }
+    end
+  end
+end
diff --git a/everest-bootstrap/lib/everest-bootstrap/ddns.rb b/everest-bootstrap/lib/everest-bootstrap/ddns.rb
new file mode 100644
index 0000000..176009a
--- /dev/null
+++ b/everest-bootstrap/lib/everest-bootstrap/ddns.rb
@@ -0,0 +1,57 @@
+require 'net/https'
+
+module RedHatDDNS
+  SERVER = 'kickstart.rdu.redhat.com'
+
+  class DDNS
+    def initialize(username, password, host)
+      @username = username
+      @password = password
+      @hostname = host
+    end
+  
+    def ddns_hash
+      @ddns_hash ||= fetch_hash
+    end
+   
+    def to_s
+      ddns_hash
+    end
+
+    def main_page
+      req = "wget -q -O- --no-check-certificate " +
+            "--http-user=#{@username} --http-password='#{@password}' " +
+            %["https://#{SERVER}/redhat-ddns/admin/"]
+
+      puts "Fetching DDNS hash..."
+      yield `#{req}`
+    end
+ 
+    def try_fetch 
+      main_page do |html|
+        /#{@hostname}.*?hash=(.*)\"/.match(html)[1] rescue nil
+      end
+    end
+
+    def request_new_hash
+      req = "wget -q -O- --no-check-certificate " +
+            "--http-user=#{@username} --http-password='#{@password}' " +
+            %["https://#{SERVER}/redhat-ddns/admin/add.php?host=#{@hostname}&_submit=Submit+Request"]
+    
+      puts "Requesting a new hash"
+      `#{req}` 
+    end
+
+    def fetch_hash(options={:tries => 3})
+      options[:tries].times do 
+        hash = try_fetch
+        return hash if hash
+        puts "Hash not found for #{@hostname}"
+        request_new_hash
+      end
+
+      $stderr.puts "Error fetching DDNS hash.  Maybe you typed your password incorrectly."
+      exit 1
+    end
+  end
+end
diff --git a/everest-bootstrap/lib/everest-bootstrap/version.rb b/everest-bootstrap/lib/everest-bootstrap/version.rb
new file mode 100644
index 0000000..a8e8aaf
--- /dev/null
+++ b/everest-bootstrap/lib/everest-bootstrap/version.rb
@@ -0,0 +1,9 @@
+module EverestBootstrap
+  module Version
+    MAJOR = 0
+    MINOR = 2
+    BUILD = 2
+
+    STRING = [MAJOR, MINOR, BUILD].join(".")
+  end
+end
diff --git a/everest-bootstrap/test/test_everest-bootstrap.rb b/everest-bootstrap/test/test_everest-bootstrap.rb
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/everest-bootstrap/test/test_everest-bootstrap.rb