More error handling

Adding a proxy pass reverse for the machine deletion redirects

2 files changed, 70 insertions, 26 deletions

diff --git a/cloudmasterd/extra/cloudmasterd.conf b/cloudmasterd/extra/cloudmasterd.conf
index 5d207ed..eee8b37 100644
--- a/cloudmasterd/extra/cloudmasterd.conf
+++ b/cloudmasterd/extra/cloudmasterd.conf
@@ -2,3 +2,4 @@ RewriteEngine On
 RewriteRule ^/$ http://%{SERVER_NAME}/cloud/status.html [NE]
 
 ProxyPass /cloud http://localhost:8107
+ProxyPassReverse /cloud http://localhost:8107
diff --git a/cloudmasterd/lib/cloudmasterd.rb b/cloudmasterd/lib/cloudmasterd.rb
index 7d9f81a..2a32879 100644
--- a/cloudmasterd/lib/cloudmasterd.rb
+++ b/cloudmasterd/lib/cloudmasterd.rb
@@ -13,6 +13,7 @@ require 'xmlrpc/client'
 require 'sqlite3'
 require 'fastthread'
 require 'open3'
+require 'pp'
 
 Camping.goes :Cloudmasterd
 Cloudmasterd.picnic!
@@ -20,29 +21,54 @@ Cloudmasterd.picnic!
 module Cloudmasterd::Helpers
   class Syncer
     def self.run
-      sync_memory
-      sync_state
-      sync_owners
+      begin
+        sync_memory
+        sync_state
+        sync_owners
+      rescue Exception => e
+        puts "ERROR: An error occured during syncing - #{e.to_s}"
+      end
+    end
+
+    def self.parse_hosts(func_stderr)
+      hosts = []
+      func_stderr.split("\n").each do |host_line|
+        # Handle a remote exception coming from a disconnected host
+        if host_line.index("remote exception") then
+          hosts.pop()
+        else
+          hosts << host_line.split(":")[1][2..-1]
+        end
+      end
+      return hosts
     end
 
     def self.sync_memory
-      # Figure out the current amount of memory
-      freemem = 0
-      `func '*' call virt freemem`.split("\n").each{|mem| freemem = freemem + mem.to_i}
-
-      memory = Cloudmasterd::Models::Memory.find(:all)
-      if memory.empty? then
-        Cloudmasterd::Models::Memory.create :free => freemem
-      else
-        memory[0].update_attribute(:free, freemem)
+      current_state = {}
+      Open3.popen3("func '*' call virt freemem") do |stdin, stdout, stderr|
+        hosts = parse_hosts(stderr.read)
+
+        # Parse out the machines that belong to each host and zip them together in a hash
+        stdout.read.split("\n").each_with_index do |memory, index|
+          current_state[hosts[index]] = memory
+        end
+      end
+
+      Cloudmasterd::Models::Cloud.transaction do
+        # Sync up the DB state - delete everything first
+        Cloudmasterd::Models::Cloud.delete_all
+
+        # Create the current entries
+        current_state.each do |host, memory|
+          Cloudmasterd::Models::Cloud.create :name => host, :memory => memory
+        end
       end
     end
 
     def self.sync_state
       current_state = {}
       Open3.popen3("func '*' call virt state") do |stdin, stdout, stderr|
-        # Parse out the hosts based on the func output (stripping off https:// and the port)
-        hosts = stderr.read.split("\n").collect{|host| host.split(" ")[1].split(":")[1][2..-1]}
+        hosts = parse_hosts(stderr.read)
 
         # Parse out the machines that belong to each host and zip them together in a hash
         stdout.read.split("\n").each_with_index do |host, index|
@@ -63,25 +89,25 @@ module Cloudmasterd::Helpers
           current_state.delete(machine.name)
         else
           # Not good - the machine in the DB isn't locateable
-          machine.update_attribute(:state, "Missing")
+          machine.update_attribute(:state, "missing")
         end
       end
 
       # We need to create an 'unknown' instance for all the remaining keys
       current_state.each do |name, machine|
-        Cloudmasterd::Models::Machine.create :name => name, :email => "Unknown", :cloud => machine[:cloud], :state => machine[:state]
+        Cloudmasterd::Models::Machine.create :name => name, :email => "unknown", :cloud => machine[:cloud], :state => machine[:state]
       end
     end
 
     def self.sync_owners
       # Sync up any unowned machines that now have an owner (handles race condition with syncing states)
       unowned_machines = {}
-      Cloudmasterd::Models::Machine.find(:all, :conditions => "email = 'Unknown'").each do |machine|
+      Cloudmasterd::Models::Machine.find(:all, :conditions => "email = 'unknown'").each do |machine|
         unowned_machines[machine.name] = machine
       end
 
       owned_machines = {}
-      Cloudmasterd::Models::Machine.find(:all, :conditions => "email <> 'Unknown'").each do |machine|
+      Cloudmasterd::Models::Machine.find(:all, :conditions => "email <> 'unknown'").each do |machine|
         owned_machines[machine.name] = machine
       end
 
@@ -159,13 +185,14 @@ module Cloudmasterd::Helpers
 end
 
 module Cloudmasterd::Models
-  class Memory < Base; end
+  class Cloud < Base; end
   class Machine < Base; end
 
   class CreateTheBasics < V 1.0
     def self.up
-      create_table :cloudmasterd_memory do |t|
-        t.column :free,   :string,  :null => false, :limit => 255
+      create_table :cloudmasterd_cloud do |t|
+        t.column :name,     :string,  :null => false, :limit => 255
+        t.column :memory,   :integer,  :null => false
       end
       create_table :cloudmasterd_machine do |t|
         t.column :name,   :string,  :null => false, :limit => 255
@@ -232,7 +259,7 @@ module Cloudmasterd::Controllers
       puts "Destroying image #{name}"
       Machine.find(:all, :conditions => "name = '#{name}'").each do |machine|
         # If the machine is missing, don't bother remotely trying to cleanup
-        unless machine.state == "Missing" then
+        unless machine.state == "missing" then
           # Destroy the virtual machine
           `func "#{machine.cloud}" call command run "virsh destroy #{machine.name}"`
           `func "#{machine.cloud}" call command run "virsh undefine #{machine.name}"`
@@ -247,7 +274,8 @@ module Cloudmasterd::Controllers
         # Delete the DB record
         machine.destroy
       end
-      redirect R(Cloud)
+
+      redirect R(Status)
     end
   end
 
@@ -258,7 +286,9 @@ module Cloudmasterd::Controllers
         @machines[email] = Machine.find(:all, :conditions => "email = '#{email}'", :order => 'cloud')
       end
 
-      @memory = Memory.find(:all)[0][:free]
+      @memory = Cloud.sum('memory')
+      @cloud_machines = Cloud.find(:all)
+
       render :status
     end
   end
@@ -286,8 +316,21 @@ module Cloudmasterd::Views
     end
 
     def status
-      div.memory! do
+      div.cloud! do
         h2 "Available cloud memory: #{@memory}"
+
+        table do
+          tr do
+            th 'Cloud Instance'
+            th 'Memory'
+          end
+          @cloud_machines.each do |machine|
+            tr do
+              td machine.name
+              td machine.memory
+            end
+          end
+        end
       end
 
       div.machines! do
@@ -307,7 +350,7 @@ module Cloudmasterd::Views
                   td machine.cloud
                   td machine.state
                   td do
-                    form(:method => :delete, :action => R(Cloudmasterd::Controllers::Koan, "#{machine.name}")) do
+                    form(:method => :delete, :action => "/cloud" + R(Cloudmasterd::Controllers::Koan, "#{machine.name}").to_s) do
                       input :type => :submit, :value => 'Destroy'
                     end
                   end