controller: configuration cleanup

Better define configuration and provide examples.

* controller: Unpack results to correct directory, load the
configuration from new place, call run_remote with proper
arguments, generate 'results.html' with result_stats script.
* ansible/run_include: Adjust to better simulate run_remote.
* ansible/fedora.yml: Adjust for fixed configuration.
* run_remote: Likewise. Also small issues with option parsing
fixed.
* config.sh.template: Moved as config/config.sh.template.
* config/config.sh.template: Copyyed from /config.sh.template,
better documented options.
* run: Fix typo - use 'while read i' instead of 'for i in'.
* config/os/EXAMPLE.sh: New file - exmaple configuration.
* private/os/EXAMPLE.yml: Likewise.
* config/hosts.template: Likewise.
* dist.include: New file with file patterns that should be
distributed to test machine.
* dist: Distribute only those files which are necessary.
* config/.gitignore: New gitignore file.

13 files changed, 129 insertions, 48 deletions

diff --git a/ansible/fedora.yml b/ansible/fedora.yml
index c22ac4b..d96ef90 100644
--- a/ansible/fedora.yml
+++ b/ansible/fedora.yml
@@ -3,34 +3,27 @@
   gather_facts: False
   hosts: localhost
   vars_files:
-  - ../private/ostack.yml
-
-  vars:
-  - keypair: praiskup-test
-  - instance_type: a4827976-e727-4135-ba59-7d5fdb9ce4e2
-  - security_group: praiskup-dbt
-  - image_id: 5fd723b7-e00d-49d3-b17b-65af842d02ab
-  - OS_AUTH_URL: http://control.os1.phx2.redhat.com:5000/v2.0
-  - OS_TENANT_ID: "{{ os_tenant_id }}"
-  - OS_USERNAME: "{{ os_username }}"
-  - OS_PASSWORD: "{{ os_nova_password }}"
-  - OS_TENANT_NAME: "Developer Experience - Packaging"
+  - "{{ opt_credsfile }}"
 
   tasks:
   - name: generate builder name
     local_action: shell echo `dd if=/dev/urandom bs=1k count=10 | md5sum ; echo DBTESTS`
     register: vm_name
 
+  - debug: msg="osusername={{ os_username }}"
+
   - name: spin it up
-    local_action: nova_compute auth_url={{OS_AUTH_URL}}
-        flavor_id={{instance_type}} image_id="{{ image_id }}" key_name={{ keypair }}
-        login_password={{OS_PASSWORD}} login_tenant_name="{{OS_TENANT_NAME}}"
-        login_username={{OS_USERNAME}} security_groups={{security_group}}
+    local_action: nova_compute auth_url={{os_auth_url}}
+        flavor_id={{os_flavor_id}} image_id="{{ os_image_id }}" key_name={{ os_keypair }}
+        login_password={{os_nova_password}} login_tenant_name="{{os_tenant_name}}"
+        login_username={{os_username}} security_groups={{os_security_group}}
         wait=yes name="{{vm_name.stdout}}"
     register: nova
 
+  - debug: msg="{{ nova.info.addresses }}"
+
     # This is ugly as hell...  Hopefully nothing will be changing.
-  - local_action: command echo "{{ nova.info.addresses["os1-internal-1425"][1].addr }}"
+  - local_action: command echo "{{ nova.info.addresses[os_network_dev][1].addr }}"
     register: machine_ip
 
   - debug: msg="{{ machine_ip.stdout }}"
diff --git a/ansible/run_include b/ansible/run_include
index 1c7cd98..2b6ccfc 100755
--- a/ansible/run_include
+++ b/ansible/run_include
@@ -10,9 +10,10 @@ export playbook=$(readlink -f "$1")
 (   set -x
     cd "$workdir"
 
-    ansible-playbook -i "$workdir/../hosts" \
+    ansible-playbook -i "$workdir/../config/hosts" \
         --extra-vars "include_file=$playbook" \
         --extra-vars "script_name=dummy" \
+        --extra-vars "opt_tmp_resultdir=/tmp/dtf-$(date +%H%M%S%N)" \
         --extra-vars "target=host" \
         "dummy-wrapper.yml"
 )
diff --git a/config.sh.template b/config.sh.template
deleted file mode 100644
index 5eae1c9..0000000
--- a/config.sh.template
+++ /dev/null
@@ -1,8 +0,0 @@
-# Where we keep all the results
-export DTF_DATABASE=/var/lib/dtf_results
-
-# SSH user/host where the generated statistics & results will be uploaded
-export DTF_PRESENTER_SSHID=userdi@fedorapeople.org
-
-# Directory on remote host where the we upload the results
-export DTF_PRESENTER_RESULTDIR=/home/fedora/praiskup/public_html/proj/postgresql-setup/tests/dtf
diff --git a/config/.gitignore b/config/.gitignore
new file mode 100644
index 0000000..57c4fcb
--- /dev/null
+++ b/config/.gitignore
@@ -0,0 +1,2 @@
+*.sh
+hosts
diff --git a/config/config.sh.template b/config/config.sh.template
new file mode 100644
index 0000000..b2b2fa5
--- /dev/null
+++ b/config/config.sh.template
@@ -0,0 +1,24 @@
+# OpenStack ID
+# ------------
+#
+# This sets up which instance of OS are the tests run against by default.
+# Based on the OS ID (for example 'EXAMPLE'), those configuration-like files are
+# searched and read:
+#
+# * private/os/example.yml
+#       File having OpenStack credentials in ansible variable file (YAML file
+#       with "key: value" lines).  See the ./private/os/EXAMPLE.yml for
+#       variables you should set up.
+#
+# * config/os/EXAMPLE.sh
+#       This file must contain basic OpenStack specific  information about VM
+#       setup on particular OS instance.
+
+export DTF_OPENSTACK_ID=dropbear
+
+# Result Database
+# ---------------
+# Directory where the testsuite should keep its (persistent) results.  Make sure
+# that this directory is backed up.
+
+export DTF_DATABASE=/var/lib/dtf_results
diff --git a/config/hosts.template b/config/hosts.template
new file mode 100644
index 0000000..6850700
--- /dev/null
+++ b/config/hosts.template
@@ -0,0 +1,2 @@
+[host]
+10.64.27.30
diff --git a/config/os/EXAMPLE.sh b/config/os/EXAMPLE.sh
new file mode 100644
index 0000000..47e1d41
--- /dev/null
+++ b/config/os/EXAMPLE.sh
@@ -0,0 +1,15 @@
+## THIS IS EXAMPLE OPENSTACK DTF CONFIGURATION ##
+
+declare -A os_flavor_ids os_image_ids
+export os_flavor_ids=[]
+export os_image_ids=[]
+
+# connection info about particular OpenStack instance
+export os_keypair="keypair-id"
+export os_security_group=default_id
+export os_network_dev=default_id
+
+# list of distro-specific images/flavors
+os_flavor_ids["fedora20"]="2"
+os_image_ids["fedora20"]="0461eef2-5921-4c01-95a6-6edf96d41b89"
+
diff --git a/controller b/controller
index 938bab5..9770497 100755
--- a/controller
+++ b/controller
@@ -1,6 +1,9 @@
 #!/bin/bash
 
-. config.sh || { echo >&2 "sorry, but config.sh not found" ; exit 1 ; }
+. config/config.sh || {
+    echo >&2 "sorry, but config/config.sh not found"
+    exit 1
+}
 
 # (still) local variables
 distro=fedora
@@ -18,31 +21,34 @@ prereq_resultdir()
 unpack_results()
 {
     local where="$1"
+    local tmp_results_dir="$2"
+
     local workdir=$(mktemp -d "/tmp/.dtf.XXXXXX") || die "can not create workdir"
 
-    pushd "$workdir"
-    tar -xf /var/tmp/dtf.tar.gz
+    { pushd "$workdir" >/dev/null \
+      && tar -xf "$tmp_results_dir/dtf.tar.gz" \
+      && pushd dtf >/dev/null
+    } || die "can not unpack results"
 
-    pushd dtf
     for i in *.log; do
-        pwd
         local task_id="${i%%.log}"
         local mydir=""
+
+        # First item in tarball is the "main" directory
         while read dirname; do
             test -z "$mydir" && mydir="$dirname"
         done <<<"$(tar xvf "$task_id.tar.gz")"
 
+        # Give the unpacked results better name
         rm "$task_id.tar.gz"
-
         mv "$mydir" "$task_id.dir"
     done
 
+    { popd && popd ; } > /dev/null
 
-    popd
-    popd
-
-    mkdir -p "$(dirname "$where")" || die "can not create resultdir"
-    mv "$workdir/dtf" "$where"
+    { mkdir -p "$(dirname "$where")" \
+      && mv "$workdir/dtf" "$where" ;
+    } || die "can not create resultdir"
 }
 
 prereq_resultdir
@@ -52,9 +58,10 @@ workdir=$(mktemp -d "/var/tmp/dtf_postgresql_setup-XXXXXX")
 ./run_remote \
     --distro="$distro" \
     --distro-version="$distro_ver" \
-    --workdir="$workdir"
+    --workdir="$workdir" \
+    --openstack-instance="$DTF_OPENSTACK_ID"
 
 resultdir="$DTF_DATABASE/$distro/$distro_ver/$arch"
-unpack_results "$resultdir/result_$starttime"
+unpack_results "$resultdir/result_$starttime" "$workdir"
 
-./runner/result_stats "$resultdir"
+./runner/result_stats "$resultdir" > "$resultdir/results.html"
diff --git a/dist b/dist
index 62938db..9c97621 100755
--- a/dist
+++ b/dist
@@ -1,10 +1,15 @@
 #!/bin/bash
 
 name=postgresql-setup-tests
-tarball=$name.tar.gz
+tarball="$name.tar.gz"
+
 test -e .git || exit 1
-what=`ls -1 | grep -v $tarball | grep -v ansible`
-mkdir $name
-cp $what $name -R
-tar -czf $tarball $name
+
+mkdir "$name"
+while read line; do
+    line=${line##postgresql-setup-tests/}
+    cp -R "$line" "$name"
+done < dist.include
+
+tar -czf $tarball --wildcards -T dist.include
 rm -rf $name
diff --git a/dist.include b/dist.include
new file mode 100644
index 0000000..3af4cfc
--- /dev/null
+++ b/dist.include
@@ -0,0 +1,5 @@
+postgresql-setup-tests/README
+postgresql-setup-tests/tasks
+postgresql-setup-tests/lib.sh
+postgresql-setup-tests/lib_pgsql.sh
+postgresql-setup-tests/run
diff --git a/private/os/EXAMPLE.yml b/private/os/EXAMPLE.yml
new file mode 100644
index 0000000..d44cbac
--- /dev/null
+++ b/private/os/EXAMPLE.yml
@@ -0,0 +1,7 @@
+## THIS IS EXAMPLE OPENSTACK CREDENTIALS FILE ##
+---
+os_auth_url: http://openstack-controller.example.com:5000/v2.0
+os_tenant_id: 8289c1553643c7adf3476fb6234562c0
+os_tenant_name: Example Develpoment Group
+os_username: jdoe
+os_nova_password: plainPasswordKeepSecrete
diff --git a/run b/run
index d4fb8d4..4e11dd5 100755
--- a/run
+++ b/run
@@ -163,7 +163,7 @@ test $dtf_option_listonly -ne 1 && prepare_resultdir
 
 dtf_resultxml_init "$dtf_resultxml_file"
 result=0
-for i in ; do
+while read i; do
     testdir=$(dirname "$i")
     run "$testdir" || result=1
 
diff --git a/run_remote b/run_remote
index 9c30f1f..47aaeb4 100755
--- a/run_remote
+++ b/run_remote
@@ -6,9 +6,12 @@ run_playbook=${run_playbook-ansible/fedora.yml}
 
 opt_workdir=/var/tmp/dbt-results
 opt_distro=fedora
+opt_openstack_instance=dropbear
 opt_distro_ver=20
 
-longopts="distro:,distro-version:,workdir:"
+die() { echo >&2 "$@" ; exit 1 ; }
+
+longopts="distro:,distro-version:,workdir:,openstack-instance:"
 ARGS=$(getopt -o "v" -l "$longopts" -n "getopt" -- "$@") \
     || exit 1
 eval set -- "$ARGS"
@@ -19,13 +22,22 @@ while true; do
         opt_distro="$2"
         shift 2
         ;;
+
     --distro-version)
         opt_distro_ver="$2"
+        shift 2
+        ;;
+
+    --openstack-instance)
+        opt_openstack_instance="$2"
+        shift 2
         ;;
+
     --workdir)
         opt_workdir="$2"
         shift 2
         ;;
+
     --)
         shift
         break
@@ -33,8 +45,24 @@ while true; do
     esac
 done
 
+credsfile="$(readlink -f "./private/os/$opt_openstack_instance.yml")"
+test -z "$credsfile" && die "--ansible-creds option must be specified"
+test ! -f "$credsfile" && die "file $credsfile not found"
+
+config_os_file="$(readlink -f "./config/os/$opt_openstack_instance.sh")"
+test ! -r "$config_os_file" && die "file $config_os_file not found"
+. "$config_os_file"
+
+config_os_id="$opt_distro$opt_distro_ver"
+
 export ANSIBLE_HOST_KEY_CHECKING=False
 ansible-playbook "$run_playbook" \
     --extra-vars "opt_distro=$opt_distro" \
     --extra-vars "opt_distro_ver=$opt_distro_ver" \
-    --extra-vars "opt_tmp_resultdir=$opt_workdir"
+    --extra-vars "opt_tmp_resultdir=$opt_workdir" \
+    --extra-vars "opt_credsfile=$credsfile" \
+    --extra-vars "os_flavor_id=${os_flavor_ids[$config_os_id]}" \
+    --extra-vars "os_image_id=${os_image_ids[$config_os_id]}" \
+    --extra-vars "os_keypair=${os_keypair}" \
+    --extra-vars "os_security_group=${os_security_group}" \
+    --extra-vars "os_network_dev=${os_network_dev}" \