1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
"""
This script concurrently builds and migrates instances. This can be useful when
troubleshooting race-conditions in virt-layer code.
Expects:
novarc to be sourced in the environment
Helper Script for Xen Dom0:
# cat /tmp/destroy_cache_vdis
#!/bin/bash
xe vdi-list | grep "Glance Image" -C1 | grep "^uuid" | awk '{print $5}' |
xargs -n1 -I{} xe vdi-destroy uuid={}
"""
import argparse
import contextlib
import multiprocessing
import subprocess
import sys
import time
DOM0_CLEANUP_SCRIPT = "/tmp/destroy_cache_vdis"
def run(cmd):
ret = subprocess.call(cmd, shell=True)
if ret != 0:
print >> sys.stderr, "Command exited non-zero: %s" % cmd
@contextlib.contextmanager
def server_built(server_name, image_name, flavor=1, cleanup=True):
run("nova boot --image=%(image_name)s --flavor=%(flavor)s"
" --poll %(server_name)s" % locals())
try:
yield
finally:
if cleanup:
run("nova delete %(server_name)s" % locals())
@contextlib.contextmanager
def snapshot_taken(server_name, snapshot_name, cleanup=True):
run("nova image-create %(server_name)s %(snapshot_name)s"
" --poll" % locals())
try:
yield
finally:
if cleanup:
run("nova image-delete %(snapshot_name)s" % locals())
def migrate_server(server_name):
run("nova migrate %(server_name)s --poll" % locals())
cmd = "nova list | grep %(server_name)s | awk '{print $6}'" % locals()
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
stdout, stderr = proc.communicate()
status = stdout.strip()
if status.upper() != 'VERIFY_RESIZE':
print >> sys.stderr, "Server %(server_name)s failed to rebuild"\
% locals()
return False
# Confirm the resize
run("nova resize-confirm %(server_name)s" % locals())
return True
def test_migrate(context):
count, args = context
server_name = "server%d" % count
cleanup = args.cleanup
with server_built(server_name, args.image, cleanup=cleanup):
# Migrate A -> B
result = migrate_server(server_name)
if not result:
return False
# Migrate B -> A
return migrate_server(server_name)
def rebuild_server(server_name, snapshot_name):
run("nova rebuild %(server_name)s %(snapshot_name)s --poll" % locals())
cmd = "nova list | grep %(server_name)s | awk '{print $6}'" % locals()
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
stdout, stderr = proc.communicate()
status = stdout.strip()
if status != 'ACTIVE':
print >> sys.stderr, "Server %(server_name)s failed to rebuild"\
% locals()
return False
return True
def test_rebuild(context):
count, args = context
server_name = "server%d" % count
snapshot_name = "snap%d" % count
cleanup = args.cleanup
with server_built(server_name, args.image, cleanup=cleanup):
with snapshot_taken(server_name, snapshot_name, cleanup=cleanup):
return rebuild_server(server_name, snapshot_name)
def _parse_args():
parser = argparse.ArgumentParser(
description='Test Nova for Race Conditions.')
parser.add_argument('tests', metavar='TESTS', type=str, nargs='*',
default=['rebuild', 'migrate'],
help='tests to run: [rebuilt|migrate]')
parser.add_argument('-i', '--image', help="image to build from",
required=True)
parser.add_argument('-n', '--num-runs', type=int, help="number of runs",
default=1)
parser.add_argument('-c', '--concurrency', type=int, default=5,
help="number of concurrent processes")
parser.add_argument('--no-cleanup', action='store_false', dest="cleanup",
default=True)
parser.add_argument('-d', '--dom0-ips',
help="IP of dom0's to run cleanup script")
return parser.parse_args()
def main():
dom0_cleanup_script = DOM0_CLEANUP_SCRIPT
args = _parse_args()
if args.dom0_ips:
dom0_ips = args.dom0_ips.split(',')
else:
dom0_ips = []
start_time = time.time()
batch_size = min(args.num_runs, args.concurrency)
pool = multiprocessing.Pool(processes=args.concurrency)
results = []
for test in args.tests:
test_func = globals().get("test_%s" % test)
if not test_func:
print >> sys.stderr, "test '%s' not found" % test
sys.exit(1)
contexts = [(x, args) for x in range(args.num_runs)]
try:
results += pool.map(test_func, contexts)
finally:
if args.cleanup:
for dom0_ip in dom0_ips:
run("ssh root@%(dom0_ip)s %(dom0_cleanup_script)s"
% locals())
success = all(results)
result = "SUCCESS" if success else "FAILED"
duration = time.time() - start_time
print "%s, finished in %.2f secs" % (result, duration)
sys.exit(0 if success else 1)
if __name__ == "__main__":
main()
|
