diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | print-backtrace.gdb | 7 | ||||
| -rw-r--r-- | tools/monitor/main.cpp | 273 | ||||
| -rw-r--r-- | tools/monitor/monitor.pro | 5 |
4 files changed, 287 insertions, 0 deletions
@@ -18,7 +18,9 @@ stamp-h1 # files generated during build *.o +*.moc .dirstamp +tools/monitor/monitor src/tmwserv-account src/tmwserv-game accountserver.exe diff --git a/print-backtrace.gdb b/print-backtrace.gdb new file mode 100644 index 0000000..2f6da3b --- /dev/null +++ b/print-backtrace.gdb @@ -0,0 +1,7 @@ +echo == BACKTRACE ==\n +bt +echo == CODE AT FRAME 0 ==\n +list +echo == FULL BACKTRACE ==\n +bt full +quit diff --git a/tools/monitor/main.cpp b/tools/monitor/main.cpp new file mode 100644 index 0000000..9674fc7 --- /dev/null +++ b/tools/monitor/main.cpp @@ -0,0 +1,273 @@ +/** + * A monitor application that runs tmwserv-account and tmwserv-game. + * (C) 2009 Thorbjørn Lindeijer + * + * When a server crashes, a gdb process is spawned to create a backtrace, + * which is then emailed. The crashed server is restarted. + */ + +#include <QtCore> + +#include <signal.h> +#include <sys/socket.h> + +/** + * A thread that keeps a server running. It restarts the server when it quits + * for some external reason, and sends out backtraces when appropriate. + */ +class ServerThread : public QThread +{ +public: + ServerThread(const QString &executable) + : mProcess(0) + , mExecutable(executable) + , mRunning(false) + {} + + void runServer(); + void stopServer(); + +protected: + void run(); + +private: + void startProcess(); + void maybeSendBacktrace(); + + QProcess *mProcess; + QString mExecutable; + QTime mLastCrash; + bool mRunning; +}; + +void ServerThread::runServer() +{ + Q_ASSERT(!mRunning); + if (!QFile::exists(mExecutable)) { + qDebug() << mExecutable << "not found"; + return; + } + mRunning = true; + start(); +} + +void ServerThread::run() +{ + mProcess = new QProcess; + startProcess(); + + while (mRunning) { + mProcess->waitForFinished(1000); + + if (mRunning && mProcess->state() == QProcess::NotRunning) { + /* The process stopped without being requested via the monitor. + * Check for crash and send report when appropriate, then restart + * the process. + */ + qDebug() << mExecutable << "terminated unexpectedly"; + maybeSendBacktrace(); + + qDebug() << "Restarting" << mExecutable << "in 3 seconds..."; + sleep(3); + startProcess(); + } + + if (!mRunning && mProcess->state() == QProcess::Running) { + // Need to shut down the process + qDebug() << "Terminating" << mExecutable; + mProcess->terminate(); + if (!mProcess->waitForFinished(3000)) { + qDebug() << mExecutable << "didn't terminate within 3 seconds, killing"; + mProcess->kill(); + } + break; + } + } + + delete mProcess; + mProcess = 0; +} + +void ServerThread::startProcess() +{ + mProcess->start(mExecutable); + + // Give the process 3 seconds to start up + if (!mProcess->waitForStarted(3000)) { + qDebug() << "Failed to start" << mExecutable; + mRunning = false; + } else { + qDebug() << mExecutable << "started"; + } +} + +void ServerThread::maybeSendBacktrace() +{ + QFile coreFile("core"); + + if (!coreFile.exists()) { + qDebug() << "No core dump found"; + return; + } + + char *receiver = getenv("CRASH_REPORT_RECEIVER"); + if (!receiver) { + qDebug() << "CRASH_REPORT_RECEIVER environment variable not set"; + return; + } + + QProcess gdb; + gdb.start(QString("gdb %1 core -q -x print-backtrace.gdb") + .arg(mExecutable)); + + if (!gdb.waitForStarted()) { + qDebug() << "Failed to launch gdb"; + coreFile.remove(); + return; + } + + if (!gdb.waitForFinished()) { + qDebug() << "gdb process is not finishing, killing"; + gdb.kill(); + coreFile.remove(); + return; + } + + coreFile.remove(); + + const QByteArray gdbOutput = gdb.readAllStandardOutput(); + qDebug() << "gdb output:\n" << gdbOutput.constData(); + + QTime current = QTime::currentTime(); + if (!mLastCrash.isNull() && mLastCrash.secsTo(current) < 60 * 10) { + qDebug() << "Sent a crash report less than 10 minutes ago, " + "dropping this one"; + return; + } + + mLastCrash = current; + + QProcess mail; + mail.start(QString("mail -s \"Crash report for %1\" %2") + .arg(mExecutable, QString::fromLocal8Bit(receiver))); + + if (!mail.waitForStarted()) { + qDebug() << "Failed to launch mail"; + return; + } + + mail.write(gdbOutput); + mail.closeWriteChannel(); + + if (mail.waitForFinished()) { + qDebug() << "Crash report sent to" << receiver; + } else { + qDebug() << "mail process is not finishing, killing"; + mail.kill(); + } +} + + +void ServerThread::stopServer() +{ + mRunning = false; + wait(); +} + +class ServerMonitor : public QObject +{ + Q_OBJECT + +public: + ServerMonitor(); + + void start() + { + mAccountServer.runServer(); + mGameServer.runServer(); + } + + static void setupUnixSignalHandlers(); + static void termSignalHandler(int); + +private slots: + void aboutToQuit() + { + mGameServer.stopServer(); + mAccountServer.stopServer(); + } + + void handleSigTerm(); + +private: + ServerThread mAccountServer; + ServerThread mGameServer; + + static int sigtermFd[2]; + + QSocketNotifier *snTerm; +}; + + +/* What follows is a bit of jumping through hoops in order to perform Qt stuff + * in response to UNIX signals. Based on documentation at: + * http://doc.trolltech.com/unix-signals.html + */ + +int ServerMonitor::sigtermFd[2]; + +ServerMonitor::ServerMonitor() + : mAccountServer("src/tmwserv-account") + , mGameServer("src/tmwserv-game") +{ + if (::socketpair(AF_UNIX, SOCK_STREAM, 0, sigtermFd)) + qFatal("Couldn't create TERM socketpair"); + + snTerm = new QSocketNotifier(sigtermFd[1], QSocketNotifier::Read, this); + connect(snTerm, SIGNAL(activated(int)), qApp, SLOT(quit())); +} + +void ServerMonitor::setupUnixSignalHandlers() +{ + struct sigaction term; + term.sa_handler = ServerMonitor::termSignalHandler; + sigemptyset(&term.sa_mask); + term.sa_flags = SA_RESTART; + + if (sigaction(SIGTERM, &term, 0)) + qFatal("Could not set TERM signal handler"); +} + +void ServerMonitor::termSignalHandler(int) +{ + char tmp = 1; + ::write(sigtermFd[0], &tmp, sizeof(tmp)); +} + +void ServerMonitor::handleSigTerm() +{ + snTerm->setEnabled(false); + char tmp; + ::read(sigtermFd[1], &tmp, sizeof(tmp)); + + QCoreApplication::quit(); + + snTerm->setEnabled(true); +} + + +int main(int argc, char *argv[]) +{ + ServerMonitor::setupUnixSignalHandlers(); + + QCoreApplication app(argc, argv); + + ServerMonitor monitor; + monitor.start(); + + QObject::connect(&app, SIGNAL(aboutToQuit()), &monitor, SLOT(aboutToQuit())); + + return app.exec(); +} + +#include "main.moc" diff --git a/tools/monitor/monitor.pro b/tools/monitor/monitor.pro new file mode 100644 index 0000000..0ae789d --- /dev/null +++ b/tools/monitor/monitor.pro @@ -0,0 +1,5 @@ +TEMPLATE = app +DEPENDPATH += . +INCLUDEPATH += . +QT -= gui +SOURCES += main.cpp |