1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
|
#
# Copyright 2009 - 2013 Clark Williams <williams@redhat.com>
# Copyright 2009 - 2013 David Sommerseth <davids@redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# For the avoidance of doubt the "preferred form" of this code is one which
# is in an open unpatent encumbered format. Where cryptographic key signing
# forms part of the process of creating an executable the information
# including keys needed to generate an equivalently functional executable
# are deemed to be part of the source code.
#
"""
Copyright (c) 2008-2013 Red Hat Inc.
Realtime verification utility
"""
__author__ = "Clark Williams <williams@redhat.com>, David Sommerseth <davids@redhat.com>"
__license__ = "GPLv2 License"
import os, signal, sys, threading, time
from datetime import datetime
from distutils import sysconfig
from modules.loads import LoadModules
from modules.measurement import MeasurementModules, MeasurementProfile
from rtevalReport import rtevalReport
from rtevalXMLRPC import rtevalXMLRPC
from Log import Log
import rtevalConfig, rtevalMailer
import version
RTEVAL_VERSION = version.RTEVAL_VERSION
sigint_received = False
def sig_handler(signum, frame):
if signum == signal.SIGINT:
global sigint_received
sigint_received = True
print "*** SIGINT received - stopping rteval run ***"
elif signum == signal.SIGTERM:
raise RuntimeError("SIGTERM received!")
class RtEval(rtevalReport):
def __init__(self, config, loadmods, measuremods, logger):
self.__version = RTEVAL_VERSION
if not isinstance(config, rtevalConfig.rtevalConfig):
raise TypeError("config variable is not an rtevalConfig object")
if not isinstance(loadmods, LoadModules):
raise TypeError("loadmods variable is not a LoadModules object")
if not isinstance(measuremods, MeasurementModules):
raise TypeError("measuremods variable is not a MeasurementModules object")
if not isinstance(logger, Log):
raise TypeError("logger variable is not an Log object")
self.__cfg = config
self.__logger = logger
self._loadmods = loadmods
self._measuremods = measuremods
self.__rtevcfg = self.__cfg.GetSection('rteval')
self.__reportdir = None
# Import SystemInfo here, to avoid DMI warnings if RtEval() is not used
from sysinfo import SystemInfo
self._sysinfo = SystemInfo(self.__rtevcfg, logger=self.__logger)
# prepare a mailer, if that's configured
if self.__cfg.HasSection('smtp'):
self.__mailer = rtevalMailer.rtevalMailer(self.__cfg.GetSection('smtp'))
else:
self.__mailer = None
if not os.path.exists(self.__rtevcfg.xslt_report):
raise RuntimeError("can't find XSL template (%s)!" % self.__rtevcfg.xslt_report)
# Add rteval directory into module search path
sys.path.insert(0, '%s/rteval' % sysconfig.get_python_lib())
# Initialise the report module
rtevalReport.__init__(self, self.__version,
self.__rtevcfg.installdir, self.__rtevcfg.annotate)
# If --xmlrpc-submit is given, check that we can access the server
if self.__rtevcfg.xmlrpc:
self.__xmlrpc = rtevalXMLRPC(self.__rtevcfg.xmlrpc, self.__logger, self.__mailer)
if not self.__xmlrpc.Ping():
if not self.__rtevcfg.xmlrpc_noabort:
print "ERROR: Could not reach XML-RPC server '%s'. Aborting." % \
self.__rtevcfg.xmlrpc
sys.exit(2)
else:
print "WARNING: Could not ping the XML-RPC server. Will continue anyway."
else:
self.__xmlrpc = None
def __show_remaining_time(self, remaining):
r = int(remaining)
days = r / 86400
if days: r = r - (days * 86400)
hours = r / 3600
if hours: r = r - (hours * 3600)
minutes = r / 60
if minutes: r = r - (minutes * 60)
print "rteval time remaining: %d days, %d hours, %d minutes, %d seconds" % (days, hours, minutes, r)
def Prepare(self, onlyload = False):
builddir = os.path.join(self.__rtevcfg.workdir, 'rteval-build')
if not os.path.isdir(builddir): os.mkdir(builddir)
# create our report directory
try:
# Only create a report dir if we're doing measurements
# or the loads logging is enabled
if not onlyload or self.__rtevcfg.logging:
self.__reportdir = self._make_report_dir(self.__rtevcfg.workdir, "summary.xml")
except Exception, e:
raise RuntimeError("Cannot create report directory (NFS with rootsquash on?) [%s]", str(e))
self.__logger.log(Log.INFO, "Preparing load modules")
params = {'workdir':self.__rtevcfg.workdir,
'reportdir':self.__reportdir,
'builddir':builddir,
'srcdir':self.__rtevcfg.srcdir,
'verbose': self.__rtevcfg.verbose,
'debugging': self.__rtevcfg.debugging,
'numcores':self._sysinfo.cpu_getCores(True),
'logging':self.__rtevcfg.logging,
'memsize':self._sysinfo.mem_get_size(),
'numanodes':self._sysinfo.mem_get_numa_nodes(),
'duration': float(self.__rtevcfg.duration),
}
self._loadmods.Setup(params)
self.__logger.log(Log.INFO, "Preparing measurement modules")
self._measuremods.Setup(params)
def __RunMeasurementProfile(self, measure_profile):
if not isinstance(measure_profile, MeasurementProfile):
raise Exception("measure_profile is not an MeasurementProfile object")
measure_start = None
(with_loads, run_parallel) = measure_profile.GetProfile()
self.__logger.log(Log.INFO, "Using measurement profile [loads: %s parallel: %s]" % (
with_loads, run_parallel))
try:
nthreads = 0
# start the loads
if with_loads:
self._loadmods.Start()
print "rteval run on %s started at %s" % (os.uname()[2], time.asctime())
print "started %d loads on %d cores" % (self._loadmods.ModulesLoaded(), self._sysinfo.cpu_getCores(True)),
if self._sysinfo.mem_get_numa_nodes() > 1:
print " with %d numa nodes" % self._sysinfo.mem_get_numa_nodes()
else:
print ""
print "Run duration: %s seconds" % str(self.__rtevcfg.duration)
# start the cyclictest thread
measure_profile.Start()
# Uleash the loads and measurement threads
report_interval = int(self.__rtevcfg.report_interval)
nthreads = with_loads and self._loadmods.Unleash() or None
self.__logger.log(Log.INFO, "Waiting 30 seconds to let load modules settle down")
time.sleep(30)
measure_profile.Unleash()
measure_start = datetime.now()
# wait for time to expire or thread to die
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
self.__logger.log(Log.INFO, "waiting for duration (%s)" % str(self.__rtevcfg.duration))
stoptime = (time.time() + float(self.__rtevcfg.duration))
currtime = time.time()
rpttime = currtime + report_interval
load_avg_checked = 5
while (currtime <= stoptime) and not sigint_received:
time.sleep(1.0)
if not measure_profile.isAlive():
stoptime = currtime
self.__logger.log(Log.WARN,
"Measurement threads did not use the full time slot. Doing a controlled stop.")
if with_loads:
if len(threading.enumerate()) < nthreads:
raise RuntimeError, "load thread died!"
if not load_avg_checked:
self._loadmods.SaveLoadAvg()
load_avg_checked = 5
else:
load_avg_checked -= 1
if currtime >= rpttime:
left_to_run = stoptime - currtime
self.__show_remaining_time(left_to_run)
rpttime = currtime + report_interval
print "load average: %.2f" % self._loadmods.GetLoadAvg()
currtime = time.time()
self.__logger.log(Log.DEBUG, "out of measurement loop")
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
except RuntimeError, e:
raise RuntimeError("appeared during measurement: %s" % e)
finally:
# stop measurement threads
measure_profile.Stop()
# stop the loads
if with_loads:
self._loadmods.Stop()
print "stopping run at %s" % time.asctime()
# wait for measurement modules to finish calculating stats
measure_profile.WaitForCompletion()
return measure_start
def Measure(self):
# Run the full measurement suite with reports
rtevalres = 0
measure_start = None
for meas_prf in self._measuremods:
mstart = self.__RunMeasurementProfile(meas_prf)
if measure_start is None:
measure_start = mstart
self._report(measure_start, self.__rtevcfg.xslt_report)
if self.__rtevcfg.sysreport:
self._sysinfo.run_sysreport(self.__reportdir)
# if --xmlrpc-submit | -X was given, send our report to the given host
if self.__xmlrpc:
retvalres = self.__xmlrpc.SendReport(self.GetXMLreport())
self._sysinfo.copy_dmesg(self.__reportdir)
self._tar_results()
return rtevalres
|