1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
|
# An implementation of generating whitelist for safe-probes
# in systemtap based on the discussion in thread:
# http://sourceware.org/ml/systemtap/2006-q3/msg00574.html
#
# Copyright (C) 2006 IBM Corp.
#
# This file is part of systemtap, and is free software. You can
# redistribute it and/or modify it under the terms of the GNU General
# Public License (GPL); either version 2, or (at your option) any
# later version.
#
# Main ideas:
# 1)Fetch a group of probe points from probes.pending, probe them
# and run some workloads(e.g. runltp) parallely meanwhile.
#
# 2)If the probe test ends without crash, those actually triggered
# probe points are moved into probes.passed and those
# untriggered are into probes.untriggered; if the probe test
# crashes the system, it will be resumed automatically after
# system reboot. Those probe points which have been triggered are
# also moved into probes.passed, but those untriggered ones are
# moved into probes.failed.
#
# 3)Repeat the above until probes.pending becomes empty, then:
# Normally, probes.pending is reinitialized from probes.failed
# and probes.untriggered, then start the next iteration;
# But if max running level (now 3) is reached, or probes.pending,
# probes.failed and probes.untriggered are all empty, stop the
# whole test.
#
# http://sourceware.org/ml/systemtap/2006-q4/msg00143.html
# http://sourceware.org/ml/systemtap/2006-q4/msg00435.html
#
set TESTNAME "whitelist"
# Because this test will take several hours (depends on workloads
# and max_running_level) and probably cause more than once system
# crashes, it is disabled by default. It's strongly suggested
# to run it manually as a single test case if necessary.
# Comment the following two lines to enable it.
untested "$TESTNAME is disabled"
return
#################################################################
# supporting constants and procedures #
#################################################################
set MAX_RUNNING_LEVEL 2
set STP_GENWHITELIST_RUNNING "/stp_genwhitelist_running"
set STAP_RESULT "probe.out"
set PROBES_ALL "probes.all"
set PROBES_PENDING "probes.pending"
set PROBES_CURRENT "probes.current"
set PROBES_PASSED "probes.passed"
set PROBES_FAILED "probes.failed"
set PROBES_UNTRIGGERED "probes.untriggered"
set systemtap_script {
global stat
probe %s {
stat[pp()] <<< 1
}
probe timer.ms(30000) {
foreach (pp in stat)
printf("%%d %%s\n", @count(stat[pp]), pp)
}
probe end {
foreach (pp in stat)
printf("%%d %%s\n", @count(stat[pp]), pp)
}
}
set benchs {
{/usr/local/ltp/runltp -t 300s -f syscalls}
{/usr/local/ltp/runltp -t 300s -f nfs}
{/usr/local/ltp/runltp -t 300s -f ipc}
{/usr/local/ltp/runltp -t 300s -f dio}
{/usr/local/ltp/runltp -t 300s -f fs}
{/usr/local/ltp/runltp -t 300s -f mm}
{/usr/local/ltp/runltp -t 300s -f tcp_cmds}
{/usr/local/ltp/runltp -t 300s -f ltp-aio-stress.part1}
}
set init_probes_all_script {
probe scheduler.*,
ioscheduler.*,
ioscheduler.*.return,
syscall.*,
syscall.*.return,
ioblock.*,
netdev.*,
vm.*,
signal.*,
signal.*.return,
udp.*,
udp.*.return,
tcp.*,
tcp.*.return,
kprocess.*,
process.*,
nfs.fop.*,
nfs.aop.*,
nfsd.proc.*,
nfsd.*,
nfs.proc.*,
scsi.*,
sunrpc.*,
generic.fop.*,
generic.fop.*.return,
vfs.*,
vfs.*.return {}
}
proc do_current_test {} {
global PROBES_CURRENT
global STAP_RESULT
global systemtap_script
if [catch {open $PROBES_CURRENT r} Infile] {
puts "Failed to open $PROBES_CURRENT"
return
}
set probepoint [gets $Infile]
while {![eof $Infile]} {
set extra [gets $Infile]
if {[string length $extra] > 0} {
set probepoint [concat "$probepoint, " $extra]
set extra [gets $Infile]
}
}
catch {close $Infile}
set testname "L[get_running_level]_Grp_[string range $probepoint 0 32]..."
if [probe_ok $probepoint] {
set script [format $systemtap_script $probepoint]
whitelist_run $testname runbenchs -e $script -o $STAP_RESULT
} else {
puts "ERROR no match in probe_ok() $testname no match"
set script [format $systemtap_script $probepoint]
exec echo $script > sc.stp
}
}
proc get_linesize {filename} {
if [file readable $filename] {
scan [exec wc -l $filename] "%d" lines
if {[info exists lines] && $lines > 0} {
return $lines
}
}
return 0
}
proc get_running_level {} {
global STP_GENWHITELIST_RUNNING
if [file readable $STP_GENWHITELIST_RUNNING] {
scan [exec cat $STP_GENWHITELIST_RUNNING] "%d" current_running_level
if {[info exists current_running_level] && $current_running_level > 0} {
return $current_running_level
}
}
return 0
}
set NO_CRASH 0
set ALREADY_CRASHED 1
proc garbage_collect {{already_crashed $NO_CRASH}} {
global STAP_RESULT
global PROBES_CURRENT
global PROBES_PASSED
global PROBES_FAILED
global PROBES_UNTRIGGERED
if {[get_linesize $PROBES_CURRENT] == 0} {
exec rm -f $PROBES_CURRENT $STAP_RESULT
return
}
if {[get_linesize $STAP_RESULT] == 0} {
if {$already_crashed} {
exec cat $PROBES_CURRENT >> $PROBES_FAILED
} else {
exec cat $PROBES_CURRENT >> $PROBES_UNTRIGGERED
}
exec rm -f $PROBES_CURRENT $STAP_RESULT
return
}
# both probes.current and probe.out are non-empty
if [catch {open $PROBES_CURRENT r} Infile] {
puts "Failed to open $PROBES_CURRENT"
exec rm -f $PROBES_CURRENT $STAP_RESULT
return
}
set probepoint [gets $Infile]
while {![eof $Infile]} {
if {[is_probed $probepoint $STAP_RESULT]} {
exec echo $probepoint >> $PROBES_PASSED
} else {
if {$already_crashed} {
exec echo $probepoint >> $PROBES_FAILED
} else {
exec echo $probepoint >> $PROBES_UNTRIGGERED
}
}
set probepoint [gets $Infile]
}
catch {close $Infile}
exec rm -f $PROBES_CURRENT $STAP_RESULT
return
}
proc incr_running_level {} {
global STP_GENWHITELIST_RUNNING
set newlevel [expr [get_running_level]+1]
if { $newlevel > 0 } {
exec echo $newlevel > $STP_GENWHITELIST_RUNNING
} else {
exec echo 0 > $STP_GENWHITELIST_RUNNING
}
return
}
proc init_probes_all {} {
global PROBES_ALL
global init_probes_all_script
catch {exec stap -p2 -e $init_probes_all_script > /tmp/whitelist_tmpfile}
catch {exec grep "^kernel.function" /tmp/whitelist_tmpfile > $PROBES_ALL }
catch {exec rm -f /tmp/whitelist_tmpfile}
if {[get_linesize $PROBES_ALL] == 0} {
return 1
}
return 0
}
proc init_running_level {} {
global STP_GENWHITELIST_RUNNING
exec echo 1 > $STP_GENWHITELIST_RUNNING
}
proc is_probed {probepoint stap_result} {
if {[string length $probepoint] == 0 || [get_linesize $stap_result] == 0 } {
return 0
}
if [catch {open $stap_result r} Resfile] {
puts "Failed to open $stap_result in is_probed() proc"
return 0
}
set probed 0
set resline [gets $Resfile]
while {![eof $Resfile]} {
if {[string match "* $probepoint" $resline]} {
set probed 1
break
}
set resline [gets $Resfile]
}
catch {close $Resfile}
return $probed
}
proc probe_ok {probepoint} {
set cmd {exec stap -p2 -e }
lappend cmd "probe $probepoint {}"
exec echo $cmd > cmdfile
return ![catch $cmd]
}
proc proper_current_size {level inputfile} {
set totalsize [get_linesize $inputfile]
switch $level {
{1} {set currentsize [expr $totalsize/9]}
{2} {set currentsize [expr $totalsize/49]}
{3} {set currentsize 50}
{4} {set currentsize 10}
default {
puts "Reached unexpected iteration level: $level"
set currentsize $totalsize
}
}
if {$currentsize <= 0} {
set currentsize 5
}
return $currentsize
}
set startup_line_in_RCLOCAL "cd $env(PWD); runtest whitelist.exp&"
set RCLOCAL "/etc/rc.d/rc.local"
proc register_service {} {
global startup_line_in_RCLOCAL
global RCLOCAL
exec sed -i -n -e "/runtest whitelist.exp/!p" $RCLOCAL
exec echo $startup_line_in_RCLOCAL >> $RCLOCAL
}
proc unregister_service {} {
global RCLOCAL
exec sed -i -n -e "/runtest whitelist.exp/!p" $RCLOCAL
}
proc whitelist_run { TEST_NAME {LOAD_GEN_FUNCTION ""} args } {
set cmd [concat {stap -DMAXSKIPPED=200000 -v } $args]
catch {eval spawn $cmd}
set stap_id $spawn_id
set failed 1
expect {
-timeout 1800
-i $stap_id -re {^Pass\ ([1234]):\ [^\r]*\r\n} {
set error_msg "pass$expect_out(1,string)";
exp_continue
}
-re {Pass\ 5:\ starting\ run.\r\n} {
set error_msg "stap runtime"
runbenchs
exec kill -INT -[exp_pid -i $stap_id]
exp_continue
}
-re {Pass\ 5:\ run\ completed} {
set failed 0
}
-re {parse\ error|semantic\ error} { set detail "$expect_out(0,string)" }
timeout { set detail "stap timeout"; exec kill -INT -[exp_pid -i $stap_id] }
eof { set failed 0 }
}
catch {close -i $stap_id}
wait -i $stap_id
if {$failed} { puts "whitelist_run failure \($detail\)" }
}
proc runbenchs {} {
global benchs
set runningcount 0
foreach bench $benchs {
set benchexec [lrange $bench 0 0]
set benchargs [join [lrange $bench 1 [llength $bench]]]
if {[file executable $benchexec]} {
catch {eval spawn $benchexec $benchargs}
set benchname($spawn_id) $benchexec
} else {
catch {eval spawn sleep 30}
set benchname($spawn_id) "sleep"
}
lappend idlist $spawn_id
incr runningcount
}
while {$runningcount > 0} {
expect {
-timeout 900
-i $idlist -re {LTP\ Version:\ LTP-([0-9])+\r\n$} {
set from $expect_out(spawn_id)
lappend benchres($from) $expect_out(buffer)
} eof {
set donepos [lsearch $idlist $expect_out(spawn_id)]
set idlist [lreplace $idlist $donepos $donepos]
incr runningcount -1
wait -i $expect_out(spawn_id)
} timeout {
break
}
}
}
}
###################################################################
# Main routine of the whole test #
###################################################################
if {[info procs installtest_p] != "" && ![installtest_p]} {
untested $TESTNAME;
return
}
if {[get_running_level] == 0} {
# Not started yet, start the whole test from the scratch
# Append the startup code to /etc/rc.d/rc.local if not yet
register_service
# Check whether probes.all is empty or not given
if {[get_linesize $PROBES_ALL] == 0} {
if {[init_probes_all] != 0} {
fail "$TESTNAME unable to initialize the probe point list"
return
}
}
# Set current_running_level as 1 to indicate a new test started
init_running_level
# Initialize intermediate files based on probe.all
exec rm -f $PROBES_PENDING $PROBES_CURRENT
exec rm -f $PROBES_PASSED $PROBES_UNTRIGGERED $PROBES_FAILED
file copy $PROBES_ALL $PROBES_PENDING
exec touch $PROBES_PASSED $PROBES_UNTRIGGERED $PROBES_FAILED
puts "Start a fresh stp_genwhitelist test."
} else {
# Maybe started already, so do some cleaning if necessary
garbage_collect $ALREADY_CRASHED
puts "Recovered from last maybe crashed probe test."
}
set current_size_const [proper_current_size [get_running_level] $PROBES_ALL]
puts "current_size_const is initialized as $current_size_const"
while {1} {
puts "Current size of probes.pending is [get_linesize $PROBES_PENDING]"
if {[get_linesize $PROBES_PENDING] == 0} {
# Check whether we need the next iteration or not
global MAX_RUNNING_LEVEL
# incr running_level for the start of a new iteration
incr_running_level
puts "Running level increased to [get_running_level]"
if {[get_running_level] > $MAX_RUNNING_LEVEL} {
puts "Exceed max running level limit."
break
} else {
puts "Current running level is [get_running_level]"
exec rm -f $PROBES_PENDING
if {[get_linesize $PROBES_FAILED] > 0} {
# Append probes.failed to probes.pending
exec cat $PROBES_FAILED >> $PROBES_PENDING
file delete $PROBES_FAILED
exec touch $PROBES_FAILED
puts "Append $PROBES_FAILED to $PROBES_PENDING"
}
if {[get_linesize $PROBES_UNTRIGGERED] > 0} {
# Append probes.untriggered to probes.pending
exec cat $PROBES_UNTRIGGERED >> $PROBES_PENDING
file delete $PROBES_UNTRIGGERED
exec touch $PROBES_UNTRIGGERED
puts "Append $PROBES_UNTRIGGERED to $PROBES_PENDING"
}
if {[get_linesize $PROBES_PENDING] == 0} {
# No more pending probe points
puts "No more iterations needed. Stopped."
break
}
# set new value of current_size_const for new iteration level
set current_size_const [proper_current_size [get_running_level] $PROBES_ALL]
puts "current_size_const is set as $current_size_const now"
continue
}
}
# Now, non-empty probes.pending should be ready
# Generate probes.current
exec rm -f $PROBES_CURRENT
exec head -n $current_size_const $PROBES_PENDING > $PROBES_CURRENT
exec tail -n+[expr $current_size_const+1] $PROBES_PENDING > /tmp/whitelist_tmpfile
exec mv /tmp/whitelist_tmpfile $PROBES_PENDING
puts "Start a probe test..."
# Do actual probe test
do_current_test
puts "Completed one probe test."
# No crash fortunately, so do some cleaning to prepare for next test
garbage_collect $NO_CRASH
}
# Congratulations for arriving here
# Remove all temporary files and unregister myself
puts "Remove all temporary files, unregister the service and return."
exec rm -f $PROBES_PENDING $PROBES_CURRENT $STP_GENWHITELIST_RUNNING
exec rm -f $STAP_RESULT /tmp/whitelist_tmpfile
unregister_service
pass "$TESTNAME completed"
return
|