summaryrefslogtreecommitdiffstats
path: root/server
diff options
context:
space:
mode:
authorDavid Sommerseth <davids@redhat.com>2009-10-26 14:25:41 +0100
committerDavid Sommerseth <davids@redhat.com>2009-10-26 14:25:41 +0100
commitedeac25984921383f4b20762882690cac5470a29 (patch)
treefa79796bab0b90179f684ec9c888913122a066c5 /server
parentb7ac36d9196a5559be8a67dd5bb2b164e64abbc5 (diff)
parent81c363403356852340f7b20eed8b127f79440373 (diff)
downloadrteval-edeac25984921383f4b20762882690cac5470a29.tar.gz
rteval-edeac25984921383f4b20762882690cac5470a29.tar.xz
rteval-edeac25984921383f4b20762882690cac5470a29.zip
Merge branch 'master_ipv4' into clark
Conflicts: Makefile rteval.spec
Diffstat (limited to 'server')
-rw-r--r--server/README.xmlrpc (renamed from server/README)21
-rwxr-xr-xserver/gen_config.sh13
-rwxr-xr-xserver/install.sh16
-rw-r--r--server/parser/Makefile13
-rw-r--r--server/parser/README134
-rw-r--r--server/parser/argparser.c147
-rw-r--r--server/parser/argparser.h28
-rw-r--r--server/parser/configparser.c182
-rw-r--r--server/parser/configparser.h38
-rw-r--r--server/parser/eurephia_nullsafe.c67
-rw-r--r--server/parser/eurephia_nullsafe.h116
-rw-r--r--server/parser/eurephia_values.c319
-rw-r--r--server/parser/eurephia_values.h61
-rw-r--r--server/parser/eurephia_values_struct.h51
-rw-r--r--server/parser/eurephia_xml.c160
-rw-r--r--server/parser/eurephia_xml.h56
-rw-r--r--server/parser/log.c235
-rw-r--r--server/parser/log.h50
-rw-r--r--server/parser/parsethread.c337
-rw-r--r--server/parser/parsethread.h45
-rw-r--r--server/parser/pgsql.c989
-rw-r--r--server/parser/pgsql.h62
-rw-r--r--server/parser/rteval_parserd.c521
-rw-r--r--server/parser/sha1.c615
-rw-r--r--server/parser/sha1.h66
-rw-r--r--server/parser/statuses.h39
-rw-r--r--server/parser/threadinfo.h44
-rw-r--r--server/parser/xmlparser.c414
-rw-r--r--server/parser/xmlparser.h48
-rw-r--r--server/parser/xmlparser.xsl (renamed from server/xmlparser.xsl)46
-rw-r--r--server/rteval_xmlrpc.py3
-rw-r--r--server/rtevaldb.py69
-rw-r--r--server/xmlparser.py146
-rw-r--r--server/xmlrpc_API1.py18
34 files changed, 4931 insertions, 238 deletions
diff --git a/server/README b/server/README.xmlrpc
index f80e41e..435d679 100644
--- a/server/README
+++ b/server/README.xmlrpc
@@ -3,12 +3,18 @@
**
The XML-RPC server has the purpose of collecting information from
-several rteval clients. Each client will get a unique system ID which
-then can be used to track how each system changes behavior on
-different kernels. All of the data in the summary.xml produced by the
-rteval script is sent over to the XML-RPC server and a copy is saved,
-default location is /var/lib/rteval. This XML file is then parsed and
-the data is stored in a database for further analysis.
+several rteval clients. All the data in the summary.xml produced by the
+rteval script is sent over to the XML-RPC server and registered in a
+submission queue. The XML-RPC server will then send back a submission
+ID to the client.
+
+A parser daemon needs to run as well. This daemon is connected to the
+same database as the XML-RPC service and it will wait for new reports in
+the submission queue to be parsed. Look into the rteval/server/parser
+directory for more information on setting up the rteval_parserd process.
+
+Each parsed report will get a unique system ID which then can be used to
+track how each system changes behavior on different kernels.
**
@@ -53,7 +59,7 @@ server.
pg_hba.conf entry example:
# TYPE DATABASE USER CIDR-ADDRESS METHOD
-hostssl rteval xmlrpc 127.0.0.1/32 md5
+hostssl rteval rtevxmlrpc 127.0.0.1/32 md5
The XML-RPC database connector will always try to connect via SSL. To
modify the default password, connect to the database with psql and
@@ -95,7 +101,6 @@ or parameters is not set.
# Paths
datadir: /var/lib/rteval
- xsltpath: /usr/share/rteval
# Database parameters
db_server: localhost
diff --git a/server/gen_config.sh b/server/gen_config.sh
new file mode 100755
index 0000000..4b57353
--- /dev/null
+++ b/server/gen_config.sh
@@ -0,0 +1,13 @@
+#/bin/sh
+
+APACHECONF="apache-rteval.conf"
+INSTALLDIR="$1"
+
+echo "Creating Apache config file: apache-rteval.conf"
+escinstpath="$(echo ${INSTALLDIR} | sed -e 's/\//\\\\\//g')"
+expr=$(echo "s/{_INSTALLDIR_}/${escinstpath}/")
+eval "sed -e ${expr} ${APACHECONF}.tpl" > ${APACHECONF}
+echo "Copy the apache apache-rteval.conf into your Apache configuration"
+echo "directory and restart your web server"
+echo
+
diff --git a/server/install.sh b/server/install.sh
index 0565d7a..5f67948 100755
--- a/server/install.sh
+++ b/server/install.sh
@@ -1,13 +1,10 @@
#!/bin/sh
-PYTHON_FILES="rteval_xmlrpc.py xmlrpc_API1.py xmlparser.py rtevaldb.py database.py"
-XSLT_FILES="xmlparser.xsl"
+PYTHON_FILES="rteval_xmlrpc.py xmlrpc_API1.py rtevaldb.py database.py"
+XSLT_FILES="parser/xmlparser.xsl"
XSLTDIR="/usr/share/rteval"
-APACHECONF="apache-rteval.conf"
-RTEVALCONF="rteval-xmlrpc.conf"
-
if [ $# != 1 ]; then
echo "$0 </var/www/html/.... full path to the directory the XML-RPC server will reside>"
exit
@@ -23,11 +20,4 @@ echo "Installing XSLT templates to ${XSLTDIR}"
cp -v ${XSLT_FILES} ${XSLTDIR}
echo
-echo "Creating Apache config file: apache-rteval.conf"
-escinstpath="$(echo ${INSTALLDIR} | sed -e 's/\//\\\\\//g')"
-expr=$(echo "s/{_INSTALLDIR_}/${escinstpath}/")
-eval "sed -e ${expr} ${APACHECONF}.tpl" > ${APACHECONF}
-echo "Copy the apache apache-rteval.conf into your Apache configuration"
-echo "directory and restart your web server"
-echo
-
+./gen_config.sh ${INSTALLDIR}
diff --git a/server/parser/Makefile b/server/parser/Makefile
new file mode 100644
index 0000000..298aec0
--- /dev/null
+++ b/server/parser/Makefile
@@ -0,0 +1,13 @@
+
+CFLAGS = -g -fPIC -I. -Wall -Werror $(shell pkg-config libxml-2.0 --cflags) $(shell pkg-config libxslt --cflags)
+LDFLAGS = -lrt -lpthread -lpq $(shell pkg-config libxml-2.0 --libs) $(shell pkg-config libxslt --libs)
+
+OBJS = argparser.o configparser.o eurephia_nullsafe.o \
+ eurephia_values.o eurephia_xml.o log.o parsethread.o pgsql.o sha1.o \
+ xmlparser.o rteval_parserd.o
+
+rteval_parserd : $(OBJS)
+ gcc -g -o $@ $^ $(LDFLAGS)
+
+clean :
+ rm -f $(OBJS) rteval_parserd *~
diff --git a/server/parser/README b/server/parser/README
new file mode 100644
index 0000000..793b35b
--- /dev/null
+++ b/server/parser/README
@@ -0,0 +1,134 @@
+**
+** rteval_parsed - the rteval XML report parser
+**
+
+The purpose of the daemon is to off load the web server from the heavy duty
+work of parsing and processing the rteval XML reports. The XML-RPC server
+will receive the reports and put the files in a queue directory on the
+filesystem and register the the submission in the database. This will notify
+the rteval_parsed that a new report has been received and it will start
+processing that file independently of the web/XML-RPC server.
+
+
+** Configure rteval_parsed
+
+This daemon uses the same configuration file as the rest of the rteval program
+suite, /etc/rteval.conf. It will parse the section named 'xmlrpc_parser'.
+
+The default values are:
+
+ - xsltpath: /usr/share/rteval
+ Defines where it can find the xmlparser.xsl XSLT template
+
+ - db_server: localhost
+ Which database server to connect to
+
+ - db_port: 5432
+ Which port to use for the database connection
+
+ - database: rteval
+ Which database to make use of.
+
+ - db_username: rtevparser
+ Which user name to use for the connection
+
+ - db_password: rtevaldb_parser
+ Which password to use for the authentication
+
+ - reportdir: /var/lib/rteval/report
+ Where to save the parsed reports
+
+
+** rteval_parserd arguments
+
+ -d | --daemon Run as a daemon
+ -l | --log <log dest> Where to put log data
+ -L | --log-level <verbosity> What to log
+ -f | --config <config file> Which configuration file to use
+ -t | --threads <num. threads> How many worker threads to start (def: 4)
+ -h | --help This help screen
+
+- Configuration file
+By default the program will look for /etc/rteval.conf. This can be
+overriden by using --config <config file>.
+
+- Logging
+When the program is started as a daemon, it will log to syslog by default.
+The default log level is 'info'. When not started as a daemon, all logging
+will go to stderr by default.
+
+The --log argument takes either 'destination' or a file name. Unknown
+destinations are treated as filenames. Valid 'destinations' are:
+
+ stderr: - Log to stderr
+ stdout: - Log to stdout
+ syslog:[facility] - Log to syslog
+ <file name> - Log to given file
+
+For syslog the default facility is 'daemon', but can be overriden by using
+one of the following facility values:
+ daemon, user and local0 to local7
+
+Log verbosity is set by the --log-level. The valid values here are:
+
+ emerg, emergency - Only log errors which causes the program to stop
+ alert - Incidents which needs immediate attention
+ crit, critical - Unexpected incidents which is not urgent
+ err, error - Parsing errors. Issues with input data
+ warn, warning - Incidents which may influence performance
+ notice - Less important warnings
+ info - General run information
+ debug - Detailed run information, incl. thread operation
+
+- Threads
+By default, the daemon will use 5 threads. One for the main threads which
+processes the submission queue and notifies the working threads. The 4 other
+threads are worker threads, which will process the received reports.
+
+Each of the worker threads will have its own connection to the database. This
+connection will be connected to the database as long as the daemon is running.
+It is therefore important that you do not have more worker threads than
+available database connections.
+
+
+** POSIX Message Queue
+
+The daemon makes use of POSIX MQ for distributing work to the worker threads.
+Each thread lives independently and polls the queue regularly for more work.
+As the POSIX MQ has a pretty safe mechanism of not duplicating messages in the
+implementation, no other locking facility is needed.
+
+On Linux, the default value for maximum messages in the queue are set to 10.
+If you receive a lot of reports and the threads do not process the queue
+quickly enough, it will fill up pretty quickly. If the queue is filled up,
+the main thread which populates the message queue will politely go to sleep
+for one minute before attempting to send new messages. To avoid this, consider
+to increase the queue size by modifying /proc/sys/fs/mqueue/msg_max.
+
+When the daemon initialises itself, it will read this file to make sure it
+uses the queue to the maximum, but not beyond that.
+
+
+** PostgreSQL features
+
+The daemon depends on the PostgreSQL database. It is written with an
+abstraction layer so it should, in theory, be possible to easily adopt it to
+different database implementation.
+
+In the current implementation, it makes use of PostgreSQL's LISTEN, NOTIFY and
+UNLISTEN features. A trigger is enabled on the submission queue table, which
+sends a NOTIFY whenever a record is inserted into the table. The rteval_parser
+daemon listens for these notifications, and will immediately poll the table
+upon such a notification.
+
+Whenever a notification is received, it will always parse all unprocessed
+reports. In addition it will also only listen for notifications when there
+are no unprocessed reports.
+
+
+** Submission queue status codes
+
+In the rteval database's submissionqueue table there is a status field. The
+daemon will only consider records with status == 0 for processing. It do not
+consider any other fields. For a better understanding of the different status
+codes, look into the file statuses.h.
diff --git a/server/parser/argparser.c b/server/parser/argparser.c
new file mode 100644
index 0000000..e9d523a
--- /dev/null
+++ b/server/parser/argparser.c
@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file argparser.c
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Thu Oct 22 13:58:46 2009
+ *
+ * @brief Generic argument parser
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <eurephia_values.h>
+#include <eurephia_nullsafe.h>
+
+
+/**
+ * Print a help screen to stdout
+ */
+void usage() {
+ printf("rteval_parserd: Parses new reports recieved via XML-RPC\n"
+ "\n"
+ "This program will wait for changes to the rteval 'submissionqueue' table.\n"
+ "When a new report is registered here, it will send this report to one of\n"
+ "the worker threads which will insert the parsed result into the database.\n"
+ "\n"
+ "** Program arguments:\n"
+ " -d | --daemon Run as a daemon\n"
+ " -l | --log <log dest> Where to put log data\n"
+ " -L | --log-level <verbosity> What to log\n"
+ " -f | --config <config file> Which configuration file to use\n"
+ " -t | --threads <num. threads> How many worker threads to start (def: 4)\n"
+ " -h | --help This help screen\n"
+ "\n"
+ "** Configuration file\n"
+ "By default the program will look for /etc/rteval.conf. This can be\n"
+ "overriden by using --config <config file>.\n"
+ "\n"
+ "** Logging\n"
+ "When the program is started as a daemon, it will log to syslog by default.\n"
+ "The default log level is 'info'. When not started as a daemon, all logging\n"
+ "will go to stderr by default.\n"
+ "\n"
+ "The --log argument takes either 'destination' or a file name. Unknown\n"
+ "destinations are treated as filenames. Valid 'destinations' are:\n"
+ "\n"
+ " stderr: - Log to stderr\n"
+ " stdout: - Log to stdout\n"
+ " syslog:[facility] - Log to syslog\n"
+ " <file name> - Log to given file\n"
+ "\n"
+ "For syslog the default facility is 'daemon', but can be overriden by using\n"
+ "one of the following facility values:\n"
+ " daemon, user and local0 to local7\n"
+ "\n"
+ "Log verbosity is set by the --log-level. The valid values here are:\n"
+ "\n"
+ " emerg, emergency - Only log errors which causes the program to stop\n"
+ " alert - Incidents which needs immediate attention\n"
+ " crit, critical - Unexpected incidents which is not urgent\n"
+ " err, error - Parsing errors. Issues with input data\n"
+ " warn, warning - Incidents which may influence performance\n"
+ " notice - Less important warnings\n"
+ " info - General run information\n"
+ " debug - Detailed run information, incl. thread operations\n"
+ "\n"
+ );
+}
+
+
+/**
+ * Parses program arguments and puts the recognised arguments into an eurephiaVALUES struct.
+ *
+ * @param argc argument counter
+ * @param argv argument string table
+ *
+ * @return Returns a pointer to an eurephiaVALUES struct. On failure, the program halts.
+ */
+eurephiaVALUES *parse_arguments(int argc, char **argv) {
+ eurephiaVALUES *args = NULL;
+ int optidx, c;
+ static struct option long_opts[] = {
+ {"log", 1, 0, 'l'},
+ {"log-level", 1, 0, 'L'},
+ {"config", 1, 0, 'f'},
+ {"threads", 1, 0, 't'},
+ {"daemon", 0, 0, 'd'},
+ {"help", 0, 0, 'h'},
+ {0, 0, 0, 0}
+ };
+
+ args = eCreate_value_space(NULL, 21);
+ eAdd_value(args, "daemon", "0");
+ eAdd_value(args, "configfile", "/etc/rteval.conf");
+ eAdd_value(args, "threads", "4");
+
+ while( 1 ) {
+ optidx = 0;
+ c = getopt_long(argc, argv, "l:L:f:t:dh", long_opts, &optidx);
+ if( c == -1 ) {
+ break;
+ }
+
+ switch( c ) {
+ case 'l':
+ eUpdate_value(args, "log", optarg, 1);
+ break;
+ case 'L':
+ eUpdate_value(args, "loglevel", optarg, 1);
+ break;
+ case 'f':
+ eUpdate_value(args, "configfile", optarg, 0);
+ break;
+ case 't':
+ eUpdate_value(args, "threads", optarg, 0);
+ break;
+ case 'd':
+ eUpdate_value(args, "daemon", "1", 0);
+ break;
+ case 'h':
+ usage();
+ exit(0);
+ }
+ }
+
+ // If logging is not configured, and it is not run as a daemon
+ // -> log to stderr:
+ if( (eGet_value(args, "log") == NULL)
+ && (atoi_nullsafe(eGet_value(args, "daemon")) == 0) ) {
+ eAdd_value(args, "log", "stderr:");
+ }
+
+ return args;
+}
diff --git a/server/parser/argparser.h b/server/parser/argparser.h
new file mode 100644
index 0000000..53214e1
--- /dev/null
+++ b/server/parser/argparser.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file argparser.h
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Thu Oct 22 13:58:46 2009
+ *
+ * @brief Generic argument parser
+ *
+ */
+
+#ifndef _RTEVAL_ARGPARSER_h
+#define _RTEVAL_ARGPARSER_h
+
+eurephiaVALUES *parse_arguments(int argc, char **argv);
+
+#endif
diff --git a/server/parser/configparser.c b/server/parser/configparser.c
new file mode 100644
index 0000000..1e68d23
--- /dev/null
+++ b/server/parser/configparser.c
@@ -0,0 +1,182 @@
+/* configparser.c - Read and parse config files
+ *
+ * This code is based on the fragments from the eurephia project.
+ *
+ * GPLv2 Copyright (C) 2009
+ * David Sommerseth <davids@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+/**
+ * @file configparser.c
+ * @author David Sommerseth <davids@redhat.com>
+ * @date 2009-10-01
+ *
+ * @brief Config file parser
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <assert.h>
+
+#include <eurephia_nullsafe.h>
+#include <eurephia_values.h>
+#include <configparser.h>
+#include <log.h>
+
+/**
+ * Parse one single configuration line into a eurephiaVALUES key/value pair. It will also ignore
+ * comment lines, and also remove the comments on the line of the configuration line so that only
+ * the key/value information is extracted.
+ *
+ * @param line Input configuration line
+ *
+ * @return eurephiaVALUES pointer containing the parsed result. On error or if no valid config
+ * line was found, NULL is returned.
+ */
+static inline eurephiaVALUES *parse_config_line(LogContext *log, const char *line) {
+ char *cp = NULL, *key = NULL, *val = NULL, *ptr = NULL;;
+ eurephiaVALUES *ret = NULL;
+
+ if( *line == '#' ) {
+ return NULL;
+ }
+
+ cp = strdup(line);
+ key = cp;
+ val = strpbrk(cp, "=:");
+ if( val == NULL ) {
+ free_nullsafe(cp);
+ return NULL;
+ }
+ *val = '\0'; val++;
+
+ // Discard comments at the end of a line
+ if( (ptr = strpbrk(val, "#")) != NULL ) {
+ *ptr = '\0';
+ }
+
+ // Left trim
+ while( ((*key == 0x20) || (*key == 0x0A) || (*key == 0x0D)) ) {
+ key++;
+ }
+ while( ((*val == 0x20) || (*val == 0x0A) || (*val == 0x0D)) ) {
+ val++;
+ }
+
+ // Right trim
+ ptr = key + strlen_nullsafe(key) - 1;
+ while( ((*ptr == 0x20) || (*ptr == 0x0A) || (*ptr == 0x0D)) && (ptr > key) ) {
+ ptr--;
+ }
+ ptr++;
+ *ptr = '\0';
+
+ ptr = val + strlen_nullsafe(val) - 1;
+ while( ((*ptr == 0x20) || (*ptr == 0x0A) || (*ptr == 0x0D)) && (ptr > val) ) {
+ ptr--;
+ }
+ ptr++;
+ *ptr = '\0';
+
+ // Put key/value into a eurephiaVALUES struct and return it
+ ret = eCreate_value_space(log, 20);
+ ret->key = strdup(key);
+ ret->val = strdup(val);
+
+ free_nullsafe(cp);
+ return ret;
+}
+
+
+static inline eurephiaVALUES *default_cfg_values(LogContext *log, eurephiaVALUES *prgargs) {
+ eurephiaVALUES *cfg = NULL, *ptr = NULL;
+
+ cfg = eCreate_value_space(log, 20);
+ eAdd_value(cfg, "datadir", "/var/lib/rteval");
+ eAdd_value(cfg, "xsltpath", "/usr/share/rteval");
+ eAdd_value(cfg, "db_server", "localhost");
+ eAdd_value(cfg, "db_port", "5432");
+ eAdd_value(cfg, "database", "rteval");
+ eAdd_value(cfg, "db_username", "rtevparser");
+ eAdd_value(cfg, "db_password", "rtevaldb_parser");
+ eAdd_value(cfg, "reportdir", "/var/lib/rteval/reports");
+
+ // Copy over the arguments to the config, update existing settings
+ for( ptr = prgargs; ptr; ptr = ptr->next ) {
+ eUpdate_value(cfg, ptr->key, ptr->val, 1);
+ }
+
+ return cfg;
+}
+
+/**
+ * Parses a section of a config file and puts it into an eurephiaVALUES key/value stack
+ *
+ * @param log Initialised log context
+ * @param prgargs Parsed command line arguments (see parse_arguments())
+ * @param section Section to read from the config file
+ *
+ * @return Returns a pointer to an eurephiaVALUES stack containing the configuration on success,
+ * otherwise NULL.
+ */
+eurephiaVALUES *read_config(LogContext *log, eurephiaVALUES *prgargs, const char *section) {
+ FILE *fp = NULL;
+ char *buf = NULL, *sectmatch = NULL, *cfgname = NULL;
+ int sectfound = 0;
+ eurephiaVALUES *cfg = NULL;
+ struct stat fi;
+
+ cfgname = eGet_value(prgargs, "configfile");
+ if( stat(cfgname, &fi) == -1 ) {
+ writelog(log, LOG_EMERG, "Could not open the config file: %s", cfgname);
+ return NULL;
+ }
+
+ if( (fp = fopen(cfgname, "r")) == NULL ) {
+ writelog(log, LOG_EMERG, "Could not open the config file: %s", cfgname);
+ return NULL;
+ }
+
+ buf = (char *) malloc_nullsafe(log, fi.st_size+2);
+ sectmatch = (char *) malloc_nullsafe(log, strlen_nullsafe(section)+4);
+ sprintf(sectmatch, "[%s]", section);
+
+ cfg = default_cfg_values(log, prgargs);
+ writelog(log, LOG_DEBUG, "Reading config file: %s", cfgname);
+ while( fgets(buf, fi.st_size, fp) != NULL ) {
+ if( strncmp(buf, "[", 1) == 0 ) {
+ sectfound = (!sectfound && (strncmp(buf, sectmatch, strlen(sectmatch)) == 0));
+ continue;
+ }
+
+ if( sectfound ) {
+ eurephiaVALUES *prm = parse_config_line(log, buf);
+ if( prm != NULL ) {
+ cfg = eUpdate_valuestruct(cfg, prm, 1);
+ }
+ }
+ };
+ free_nullsafe(buf);
+ free_nullsafe(sectmatch);
+ fclose(fp); fp = NULL;
+
+ return cfg;
+}
diff --git a/server/parser/configparser.h b/server/parser/configparser.h
new file mode 100644
index 0000000..54a904b
--- /dev/null
+++ b/server/parser/configparser.h
@@ -0,0 +1,38 @@
+/* configparser.h - Read and parse config files
+ *
+ * This code is based on the fragments from the eurephia project.
+ *
+ * GPLv2 Copyright (C) 2009
+ * David Sommerseth <davids@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+/**
+ * @file configparser.h
+ * @author David Sommerseth <davids@redhat.com>
+ * @date 2009-10-01
+ *
+ * @brief Config file parser
+ *
+ */
+
+#ifndef _CONFIGPARSER_H
+#define _CONFIGPARSER_H
+
+eurephiaVALUES *read_config(LogContext *log, eurephiaVALUES *prgargs, const char *section);
+
+#endif
diff --git a/server/parser/eurephia_nullsafe.c b/server/parser/eurephia_nullsafe.c
new file mode 100644
index 0000000..e6dad0e
--- /dev/null
+++ b/server/parser/eurephia_nullsafe.c
@@ -0,0 +1,67 @@
+/* eurephia_nullsafe.c
+ *
+ * standard C string functions, which is made NULL safe by checking
+ * if input value is NULL before performing the action.
+ *
+ * This version is modified to work outside the eurephia project.
+ *
+ * GPLv2 only - Copyright (C) 2009
+ * David Sommerseth <dazo@users.sourceforge.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+/**
+ * @file eurephia_nullsafe.c
+ * @author David Sommerseth <dazo@users.sourceforge.net>
+ * @date 2009-09-07
+ *
+ * @brief standard C string functions, which is made NULL safe by checking
+ * if input value is NULL before performing the action.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <log.h>
+
+#if __GNUC__ >= 3
+#define __malloc__ __attribute__((malloc))
+#else /* If not GCC 3 or newer, disable optimisations */
+#define __malloc__
+#endif
+
+/**
+ * This replaces the use of malloc() and memset(). This function uses calloc
+ * internally, which results in the memory region being zero'd by the kernel
+ * on memory allocation.
+ *
+ * @param log Log context
+ * @param sz size of the memory region being allocated
+ *
+ * @return Returns a void pointer to the memory region on success, otherwise NULL
+ */
+__malloc__ void *malloc_nullsafe(LogContext *log, size_t sz) {
+ void *buf = NULL;
+
+ buf = calloc(1, sz); /* Using calloc, also gives a zero'd memory region */
+ if( !buf ) {
+ writelog(log, LOG_EMERG, "Could not allocate memory region for %ld bytes", sz);
+ exit(9);
+ }
+ return buf;
+}
diff --git a/server/parser/eurephia_nullsafe.h b/server/parser/eurephia_nullsafe.h
new file mode 100644
index 0000000..e0b2f32
--- /dev/null
+++ b/server/parser/eurephia_nullsafe.h
@@ -0,0 +1,116 @@
+/* eurephia_nullsafe.h
+ *
+ * standard C string functions, which is made NULL safe by checking
+ * if input value is NULL before performing the action.
+ *
+ * This version is modified to work outside the eurephia project.
+ *
+ * GPLv2 only - Copyright (C) 2008, 2009
+ * David Sommerseth <dazo@users.sourceforge.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+/**
+ * @file eurephia_nullsafe.h
+ * @author David Sommerseth <dazo@users.sourceforge.net>
+ * @date 2008-08-06
+ *
+ * @brief standard C string functions, which is made NULL safe by checking
+ * if input value is NULL before performing the action.
+ *
+ */
+
+#ifndef EUREPHIA_NULLSAFE_H_
+#define EUREPHIA_NULLSAFE_H_
+
+#include <log.h>
+
+/**
+ * atoi() wrapper. Converts any string into a integer
+ *
+ * @param str Input string
+ *
+ * @return Returns integer
+ */
+#define atoi_nullsafe(str) (str != NULL ? atoi(str) : 0)
+
+
+/**
+ * strdup() wrapper. Duplicates the input string.
+ *
+ * @param str Input string to be duplicated
+ *
+ * @return Returns a pointer to the duplicate (char *) on success, NULL otherwise.
+ * If input was NULL, NULL is returned.
+ */
+#define strdup_nullsafe(str) (str != NULL ? strdup(str) : NULL)
+
+
+/**
+ * Wrapper macro, which appends a string to a destination string without exceeding the size
+ * of the destination buffer.
+ *
+ * @param dest Pointer to the destination buffer
+ * @param src Pointer to the value being concatenated to the destination string.
+ * @param size Size of the destination buffer
+ */
+#define append_str(dest, src, size) strncat(dest, src, (size - strlen_nullsafe(dest)))
+
+
+/**
+ * strlen() wrapper. Returns the length of a string
+ *
+ * @param str Input string
+ *
+ * @return Returns int with length of string. If input is NULL, it returns 0.
+ */
+#define strlen_nullsafe(str) (str != NULL ? strlen(str) : 0)
+
+
+void *malloc_nullsafe(LogContext *, size_t);
+
+/**
+ * Null safe free(). It will not attempt to free a pointer which is NULL.
+ *
+ * @param ptr Pointer to the memory region being freed.
+ *
+ */
+#define free_nullsafe(ptr) if( ptr ) { free(ptr); ptr = NULL; }
+
+
+/**
+ * Function which will return a default string value if no input data was provided.
+ *
+ * @param str Input string
+ * @param defstr Default string
+ *
+ * @return Returns the pointer to the input string if the string length > 0. Otherwise it
+ * will return a pointer to the default string.
+ */
+#define defaultValue(str, defstr) (strlen_nullsafe(str) == 0 ? defstr : str)
+
+
+/**
+ * Function which will return a default integer value if no input data was provided.
+ *
+ * @param ival input integer value
+ * @param defval default integer value
+ *
+ * @return Returns the ival value if it is > 0, otherwise defval value is returned.
+ */
+#define defaultIntValue(ival, defval) (ival == 0 ? defval : ival)
+#endif /* !EUREPHIA_NULLSAFE_H_ */
diff --git a/server/parser/eurephia_values.c b/server/parser/eurephia_values.c
new file mode 100644
index 0000000..dbdbc13
--- /dev/null
+++ b/server/parser/eurephia_values.c
@@ -0,0 +1,319 @@
+/* eurephia_values.c -- Generic interface for processing key->value pairs
+ *
+ * This version is modified to work outside the eurephia project.
+ *
+ * GPLv2 only - Copyright (C) 2008
+ * David Sommerseth <dazo@users.sourceforge.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @file eurephia_values.c
+ * @author David Sommerseth <dazo@users.sourceforge.net>
+ * @date 2008-08-06
+ *
+ * @brief Generic interface for handling key->value pairs
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include <eurephia_nullsafe.h>
+#include <eurephia_values_struct.h>
+
+/**
+ * Internal function. Makes sure a eurephiaVALUES stack is freed up
+ *
+ * @param vls Pointer to a eurephiaVALUES stack.
+ */
+static inline void do_free_vals(eurephiaVALUES *vls) {
+ if( vls->next != NULL ) {
+ do_free_vals(vls->next);
+ }
+ free_nullsafe(vls->key);
+ free_nullsafe(vls->val);
+ free_nullsafe(vls);
+}
+
+
+/**
+ * Function for freeing up an eurephiaVALUES stack. This function is normally not called
+ * directly, but usually via the eFree_values(...) macro.
+ *
+ * @param vls Pointer to a eurephiaVALUES stack to be freed.
+ */
+void eFree_values_func(eurephiaVALUES *vls) {
+ if( (vls == NULL) ) {
+ return;
+ }
+ do_free_vals(vls);
+}
+
+
+/**
+ * Retrieve an eurephiaVALUES element for a given value key
+ *
+ * @param vls Pointer to the eurephiaVALUES stack where to search for the element
+ * @param key String containing the key name of the value requested.
+ *
+ * @return Returns an eurephiaVALUES element on success, otherwise NULL.
+ */
+eurephiaVALUES *eGet_valuestruct(eurephiaVALUES *vls, const char *key)
+{
+ eurephiaVALUES *ptr = NULL;
+
+ if( (vls == NULL) || (key == NULL) ) {
+ return NULL;
+ }
+
+ ptr = vls;
+ while( ptr != NULL ) {
+ if( (ptr->key != NULL) && (strcmp(key, ptr->key) == 0) ) {
+ return ptr;
+ }
+ ptr = ptr->next;
+ }
+ return NULL;
+}
+
+
+/**
+ * Retrieves the value of a given key from an eurephiaVALUES stack.
+ *
+ * @param vls Pointer to an eurephiaVALUES stack where to search for the value
+ * @param key String containing the key name of the value requested
+ *
+ * @return Returns a string (char *) with the requested value if found, otherwise NULL.
+ */
+char *eGet_value(eurephiaVALUES *vls, const char *key)
+{
+ eurephiaVALUES *ptr = NULL;
+
+ ptr = eGet_valuestruct(vls, key);
+ return (ptr != NULL ? ptr->val : NULL);
+}
+
+
+/**
+ * Creates a new eurephiaVALUES stack
+ *
+ * @param log Log context
+ * @param evgid int value, giving the stack an ID number. Useful when looking through log files later on.
+ *
+ * @return Returns an empty eurephiaVALUES struct on success, otherwise NULL.
+ */
+eurephiaVALUES *eCreate_value_space(LogContext *log, int evgid)
+{
+ eurephiaVALUES *ptr = NULL;
+
+ ptr = (eurephiaVALUES *) malloc_nullsafe(log, sizeof(eurephiaVALUES) + 2);
+ ptr->log = log;
+ ptr->evgid = evgid;
+ return ptr;
+}
+
+
+/**
+ * Adds a new eurephiaVALUES stack to another eurephiaVALUES stack. If the evgid value differs, it will
+ * be overwritten with the value of the destination stack.
+ *
+ * @param vls Destination eurephiaVALUES stack
+ * @param newval Source eurephiaVALUES stack
+ */
+void eAdd_valuestruct(eurephiaVALUES *vls, eurephiaVALUES *newval) {
+ eurephiaVALUES *ptr = NULL;
+ int vid = 0;
+
+ assert(vls != NULL);
+
+ if( (vls->key == NULL) && (vls->val == NULL) && (vls->next == NULL) && (vls->evid == 0)) {
+ // Update header record if it is empty, by copying newval record. Free newval afterwards
+ vls->key = strdup(newval->key);
+ vls->val = strdup(newval->val);
+ vls->evid = 0;
+ vls->next = NULL;
+ do_free_vals(newval);
+ } else {
+ // Add values to the value chain, loop to the end and append it
+ ptr = vls;
+ while( ptr->next != NULL ) {
+ ptr = ptr->next;
+ vid = (vid > ptr->evid ? vid : ptr->evid);
+ }
+ newval->evid = vid+1; // Increase the value counter
+ newval->evgid = ptr->evgid;
+ ptr->next = newval;
+ }
+}
+
+
+/**
+ * Adds a new key/value pair to an eurephiaVALUES stack
+ *
+ * @param vls Destination eurephiaVALUES stack
+ * @param key Key name for the value being stored
+ * @param val Value to be stored
+ */
+void eAdd_value(eurephiaVALUES *vls, const char *key, const char *val)
+{
+ eurephiaVALUES *ptr = NULL;
+
+ assert(vls != NULL);
+
+ // Allocate buffer and save values
+ ptr = eCreate_value_space(vls->log, vls->evid);
+ if( ptr == NULL ) {
+ writelog(vls->log, LOG_EMERG, "Failed to add value to the value chain");
+ exit(9);
+ }
+ ptr->key = strdup_nullsafe(key);
+ ptr->val = strdup_nullsafe(val);
+ ptr->evgid = vls->evgid;
+
+ // Add value struct to the chain
+ eAdd_valuestruct(vls, ptr);
+}
+
+
+/**
+ * Updates the value of a key in a values stack
+ *
+ * @param vls eurephiaVALUES key/value stack to update
+ * @param key String with key name to update
+ * @param newval String with the new value
+ * @param addunkn Add unknown keys. If set to 1, if the key is not found it will add a new key
+ */
+void eUpdate_value(eurephiaVALUES *vls, const char *key, const char *newval, const int addunkn) {
+ eurephiaVALUES *ptr = NULL;
+
+ assert( (vls != NULL) && (key != NULL) );
+
+ ptr = eGet_valuestruct(vls, key);
+ if( ptr ) {
+ free_nullsafe(ptr->val);
+ ptr->val = strdup_nullsafe(newval);
+ } else if( addunkn == 1 ) {
+ eAdd_value(vls, key, newval);
+ }
+}
+
+
+/**
+ * Updates a value struct element based on another value struct element contents (key/value)
+ *
+ * @param vls eurephiaVALUES key/value stack to update
+ * @param newval eurephiaVALUES element with the new value
+ * @param addunkn Add unknown keys. If set to 1, if the key is not found it will add a new key
+ *
+ * @return Returns a pointer to the first element in the chain. If the element being updated
+ * was the first element in the old chain, the first element will be a new element with a
+ * new address.
+ */
+eurephiaVALUES *eUpdate_valuestruct(eurephiaVALUES *vls, eurephiaVALUES *newval, const int addunkn) {
+ eurephiaVALUES *ptr = NULL, *prevptr = NULL;
+
+ assert( (vls != NULL) && (newval != NULL) && (newval->key != NULL) );
+
+ prevptr = vls;
+ for( ptr = vls; ptr != NULL; ptr = ptr->next ) {
+ if( (ptr->key != NULL) && (strcmp(newval->key, ptr->key) == 0) ) {
+ newval->evgid = ptr->evgid;
+ newval->evid = ptr->evid;
+ newval->next = ptr->next;
+ ptr->next = NULL;
+ if( ptr == vls ) {
+ // If the element found is the first one, do special treatment
+ eFree_values_func(ptr);
+ return newval;
+ } else {
+ prevptr->next = newval;
+ eFree_values_func(ptr);
+ return vls;
+ }
+ }
+ prevptr = ptr;
+ }
+
+ if( addunkn == 1 ) {
+ eAdd_valuestruct(vls, newval);
+ }
+ return vls;
+}
+
+
+/**
+ * Removes the key/value pair identified by evgid and evid from the given eurephiaVALUES chain
+ *
+ * @param vls Pointer to an eurephiaVALUES chain with the data
+ * @param evgid Group ID of the chain
+ * @param evid Element ID of the chain element to be removed
+ *
+ * @return Returns a pointer to the chain. The pointer is only changed if the first element in the
+ * chain is deleted
+ */
+eurephiaVALUES *eRemove_value(eurephiaVALUES *vls, unsigned int evgid, unsigned int evid) {
+ eurephiaVALUES *ptr = NULL, *prev_ptr = NULL;
+ int found = 0;
+
+ // Find the value element
+ for( ptr = vls; ptr != NULL; ptr = ptr->next ) {
+ if( (ptr->evgid == evgid) && (ptr->evid == evid) ) {
+ found = 1;
+ break;
+ }
+ prev_ptr = ptr;
+ }
+
+ if( !found ) {
+ return vls;
+ }
+
+ if( ptr != vls ) {
+ prev_ptr->next = ptr->next;
+ ptr->next = NULL;
+ eFree_values_func(ptr);
+ return vls;
+ } else {
+ prev_ptr = ptr->next;
+ ptr->next = NULL;
+ eFree_values_func(ptr);
+ return prev_ptr;
+ }
+}
+
+
+/**
+ * Counts number of elements in an eurephiaVALUES chain.
+ *
+ * @param vls eurephiaVALUES pointer to be counted
+ *
+ * @return Returns number of elements found.
+ */
+unsigned int eCount(eurephiaVALUES *vls) {
+ eurephiaVALUES *ptr = NULL;
+ unsigned int c = 0;
+
+ if( vls == NULL ) {
+ return 0;
+ }
+ for(ptr = vls; ptr != NULL; ptr = ptr->next ) {
+ c++;
+ }
+ return c;
+}
diff --git a/server/parser/eurephia_values.h b/server/parser/eurephia_values.h
new file mode 100644
index 0000000..c63eb4c
--- /dev/null
+++ b/server/parser/eurephia_values.h
@@ -0,0 +1,61 @@
+/* eurephia_values.h -- Generic interface for processing key->value pairs
+ *
+ * This version is modified to work outside the eurephia project.
+ *
+ * GPLv2 only - Copyright (C) 2008
+ * David Sommerseth <dazo@users.sourceforge.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+/**
+ * @file eurephia_values.h
+ * @author David Sommerseth <dazo@users.sourceforge.net>
+ * @date 2008-08-06
+ *
+ * @brief Generic interface for handling key->value pairs
+ *
+ */
+
+#include <eurephia_values_struct.h>
+
+#ifndef EUREPHIA_VALUES_H_
+#define EUREPHIA_VALUES_H_
+
+
+eurephiaVALUES *eGet_valuestruct(eurephiaVALUES *vls, const char *key);
+char *eGet_value(eurephiaVALUES *vls, const char *key);
+
+eurephiaVALUES *eCreate_value_space(LogContext *log, int evid);
+
+void eAdd_valuestruct(eurephiaVALUES *vls, eurephiaVALUES *newval);
+void eAdd_value(eurephiaVALUES *vls, const char *key, const char *val);
+void eUpdate_value(eurephiaVALUES *vls, const char *key, const char *newval, const int addunkn);
+eurephiaVALUES *eUpdate_valuestruct(eurephiaVALUES *vls, eurephiaVALUES *newval, const int addunkn);
+eurephiaVALUES *eRemove_value(eurephiaVALUES *vls, unsigned int evgid, unsigned int evid);
+unsigned int eCount(eurephiaVALUES *vls);
+
+/**
+ * Front-end function for eFree_values_func(). Frees eurephiaVALUES pointer chain and
+ * sets the pointer to NULL.
+ *
+ * @param v eurephiaVALUES pointer which is being freed.
+ *
+ */
+#define eFree_values(v) { eFree_values_func(v); v = NULL; }
+void eFree_values_func(eurephiaVALUES *vls);
+
+#endif /* !EUREPHIA_VALUES_H_ */
diff --git a/server/parser/eurephia_values_struct.h b/server/parser/eurephia_values_struct.h
new file mode 100644
index 0000000..7eb8b74
--- /dev/null
+++ b/server/parser/eurephia_values_struct.h
@@ -0,0 +1,51 @@
+/* eurephia_values.h -- eurephiaVALUES struct typedef
+ *
+ * GPLv2 only - Copyright (C) 2008
+ * David Sommerseth <dazo@users.sourceforge.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+/**
+ * @file eurephia_values_struct.h
+ * @author David Sommerseth <dazo@users.sourceforge.net>
+ * @date 2008-11-05
+ *
+ * @brief Definition of the eurephiaVALUES struct
+ *
+ */
+
+#ifndef EUREPHIA_VALUES_STRUCT_H_
+# define EUREPHIA_VALUES_STRUCT_H_
+
+#include <log.h>
+
+/**
+ * eurephiaVALUES is a pointer chain with key/value pairs. If having several
+ * such pointer chains, they can be given different group IDs to separate them,
+ * which is especially useful during debugging.
+ *
+ */
+typedef struct __eurephiaVALUES {
+ LogContext *log; /**< Pointer to an established log context, used for logging */
+ unsigned int evgid; /**< Group ID, all elements in the same chain should have the same value */
+ unsigned int evid; /**< Unique ID per element in a pointer chain */
+ char *key; /**< The key name of a value */
+ char *val; /**< The value itself */
+ struct __eurephiaVALUES *next; /**< Pointer to the next element in the chain. NULL == end of chain */
+} eurephiaVALUES;
+
+#endif /* !EUREPHIA_VALUES_STRUCT_H_ */
diff --git a/server/parser/eurephia_xml.c b/server/parser/eurephia_xml.c
new file mode 100644
index 0000000..c8c389e
--- /dev/null
+++ b/server/parser/eurephia_xml.c
@@ -0,0 +1,160 @@
+/* eurephia_xml.c -- Generic helper functions for XML parsing
+ *
+ * This version is modified to work outside the eurephia project.
+ *
+ * GPLv2 only - Copyright (C) 2008, 2009
+ * David Sommerseth <dazo@users.sourceforge.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+/**
+ * @file eurephia_xml.c
+ * @author David Sommerseth <dazo@users.sourceforge.net>
+ * @date 2008-12-15
+ *
+ * @brief Generic XML parser functions
+ *
+ *
+ */
+
+#include <stdarg.h>
+#include <string.h>
+#include <assert.h>
+
+#include <libxml/tree.h>
+#include <libxml/xmlsave.h>
+#include <libxml/xmlstring.h>
+
+#include <eurephia_nullsafe.h>
+
+
+/**
+ * Retrieves a given XML node attribute/property
+ *
+ * @param attr xmlAttr pointer from an xmlNode pointer.
+ * @param key The attribute name to search for
+ *
+ * @return The value of the found attribute. If not found, NULL is returned.
+ */
+char *xmlGetAttrValue(xmlAttr *attr, const char *key) {
+ xmlAttr *aptr;
+ xmlChar *x_key = NULL;
+
+ x_key = xmlCharStrdup(key);
+ assert( x_key != NULL );
+
+ for( aptr = attr; aptr != NULL; aptr = aptr->next ) {
+ if( xmlStrcmp(aptr->name, x_key) == 0 ) {
+ free_nullsafe(x_key);
+ return (char *)(aptr->children != NULL ? aptr->children->content : NULL);
+ }
+ }
+ free_nullsafe(x_key);
+ return NULL;
+}
+
+
+/**
+ * Loops through a xmlNode chain to look for a given tag. The search is not recursive.
+ *
+ * @param node xmlNode pointer where to look
+ * @param key the name of the XML tag to find
+ *
+ * @return xmlNode pointer to the found xmlNode. NULL is returned if not found.
+ */
+xmlNode *xmlFindNode(xmlNode *node, const char *key) {
+ xmlNode *nptr = NULL;
+ xmlChar *x_key = NULL;
+
+ if( (node == NULL) || (node->children == NULL) ) {
+ return NULL;
+ }
+
+ x_key = xmlCharStrdup(key);
+ assert( x_key != NULL );
+
+ for( nptr = node->children; nptr != NULL; nptr = nptr->next ) {
+ if( xmlStrcmp(nptr->name, x_key) == 0 ) {
+ free_nullsafe(x_key);
+ return nptr;
+ }
+ }
+ free_nullsafe(x_key);
+ return NULL;
+}
+
+
+/**
+ * Return the text content of a given xmlNode
+ *
+ * @param n xmlNode to extract the value from.
+ *
+ * @return returns a char pointer with the text contents of an xmlNode.
+ */
+inline char *xmlExtractContent(xmlNode *n) {
+ return (char *) (((n != NULL) && (n->children != NULL)) ? n->children->content : NULL);
+}
+
+
+/**
+ * Get the text contents of a given xmlNode
+ *
+ * @param node An xmlNode pointer where to look for the contents
+ * @param key Name of the tag to retrieve the content of.
+ *
+ * @return Returns a string with the text content, if the node is found. Otherwise, NULL is returned.
+ */
+inline char *xmlGetNodeContent(xmlNode *node, const char *key) {
+ return xmlExtractContent(xmlFindNode(node, key));
+}
+
+
+/**
+ * Serialises an xmlNode to a string
+ *
+ * @param log Log context
+ * @param node Input XML node to be serialised
+ *
+ * @return Returns a pointer to a new buffer containing the serialised data. This buffer must be freed
+ * after usage
+ */
+char *xmlNodeToString(LogContext *log, xmlNode *node) {
+ xmlBuffer *buf = NULL;
+ xmlSaveCtxt *serctx = NULL;
+ char *ret = NULL;
+
+ if( node == NULL ) {
+ writelog(log, LOG_ALERT, "xmlNodeToString: Input data is NULL");
+ return NULL;
+ }
+
+ buf = xmlBufferCreate();
+ assert( buf != NULL );
+
+ serctx = xmlSaveToBuffer(buf, "UTF-8", XML_SAVE_NO_EMPTY|XML_SAVE_NO_DECL);
+ assert( serctx != NULL );
+
+ if( xmlSaveTree(serctx, node) < 0 ) {
+ writelog(log, LOG_ALERT, "xmlNodeToString: Failed to serialise xmlNode");
+ return NULL;
+ }
+ xmlSaveClose(serctx);
+
+ ret = strdup_nullsafe((char *) xmlBufferContent(buf));
+ xmlBufferFree(buf);
+ return ret;
+}
diff --git a/server/parser/eurephia_xml.h b/server/parser/eurephia_xml.h
new file mode 100644
index 0000000..dea72eb
--- /dev/null
+++ b/server/parser/eurephia_xml.h
@@ -0,0 +1,56 @@
+/* eurephia_xml.h -- Generic helper functions for XML parsing
+ *
+ * This version is modified to work outside the eurephia project.
+ *
+ * GPLv2 only - Copyright (C) 2008
+ * David Sommerseth <dazo@users.sourceforge.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+/**
+ * @file eurephia_xml.h
+ * @author David Sommerseth <dazo@users.sourceforge.net>
+ * @date 2008-12-15
+ *
+ * @brief Generic XML parser functions
+ *
+ */
+
+
+#ifndef EUREPHIA_XML_H_
+#define EUREPHIA_XML_H_
+
+#include <stdarg.h>
+
+#include <libxml/tree.h>
+
+/**
+ * Simple iterator macro for iterating xmlNode pointers
+ *
+ * @param start Pointer to an xmlNode where to start iterating
+ * @param itn An xmlNode pointer which will be used for the iteration.
+ */
+#define foreach_xmlnode(start, itn) for( itn = start; itn != NULL; itn = itn->next )
+
+char *xmlGetAttrValue(xmlAttr *properties, const char *key);
+xmlNode *xmlFindNode(xmlNode *node, const char *key);
+
+inline char *xmlExtractContent(xmlNode *n);
+inline char *xmlGetNodeContent(xmlNode *node, const char *key);
+char *xmlNodeToString(LogContext *log, xmlNode *node);
+
+#endif /* !EUREPHIA_XML_H_ */
diff --git a/server/parser/log.c b/server/parser/log.c
new file mode 100644
index 0000000..b7c2ad8
--- /dev/null
+++ b/server/parser/log.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file log.c
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Wed Oct 21 11:38:51 2009
+ *
+ * @brief Generic log functions
+ *
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <pthread.h>
+#include <syslog.h>
+
+#include <eurephia_nullsafe.h>
+#include <log.h>
+
+/**
+ * Maps defined log level strings into syslog
+ * compatible LOG_* integer values
+ */
+static struct {
+ const char *priority_str;
+ const int prio_level;
+} syslog_prio_map[] = {
+ {"emerg", LOG_EMERG},
+ {"emergency", LOG_EMERG},
+ {"alert", LOG_ALERT},
+ {"crit", LOG_CRIT},
+ {"critical", LOG_CRIT},
+ {"err", LOG_ERR},
+ {"error", LOG_ERR},
+ {"warning", LOG_WARNING},
+ {"warn", LOG_WARNING},
+ {"notice", LOG_NOTICE},
+ {"info", LOG_INFO},
+ {"debug", LOG_DEBUG},
+ {NULL, 0}
+};
+
+
+/**
+ * Initialises a log context. It parses the log destination and log level and
+ * prepares a context which can be used by writelog()
+ *
+ * @param logdest String containing either syslog:[facility], stderr: or stdout:, or a file name.
+ * @param loglvl Defines the log level. Can be one of the values defined in syslog_prio_map.
+ *
+ * @return Returns a pointer to a log context on success, otherwise NULL.
+ */
+LogContext *init_log(const char *logdest, const char *loglvl) {
+ LogContext *logctx = NULL;
+ int i;
+
+ logctx = (LogContext *) calloc(1, sizeof(LogContext)+2);
+ assert( logctx != NULL);
+
+ logctx->logfp = NULL;
+
+ // Get the int value of the log level string
+ logctx->verbosity = -1;
+ if( loglvl ) {
+ for( i = 0; syslog_prio_map[i].priority_str; i++ ) {
+ if( strcasecmp(loglvl, syslog_prio_map[i].priority_str) == 0 ) {
+ logctx->verbosity = syslog_prio_map[i].prio_level;
+ break;
+ }
+ }
+ }
+
+ // If log level is not set, set LOG_INFo as default
+ if( logctx->verbosity == -1 ) {
+ logctx->verbosity = LOG_INFO;
+ }
+
+ if( logdest == NULL ) {
+ logctx->logtype = ltSYSLOG;
+ openlog("rteval_parserd", LOG_PID, LOG_DAEMON);
+ } else {
+ if( strncmp(logdest, "syslog:", 7) == 0 ) {
+ const char *fac = logdest+7;
+ int facid = LOG_DAEMON;
+
+ if( strcasecmp(fac, "local0") == 0 ) {
+ facid = LOG_LOCAL0;
+ } else if( strcasecmp(fac, "local1") == 0 ) {
+ facid = LOG_LOCAL1;
+ } else if( strcasecmp(fac, "local2") == 0 ) {
+ facid = LOG_LOCAL2;
+ } else if( strcasecmp(fac, "local3") == 0 ) {
+ facid = LOG_LOCAL3;
+ } else if( strcasecmp(fac, "local4") == 0 ) {
+ facid = LOG_LOCAL4;
+ } else if( strcasecmp(fac, "local5") == 0 ) {
+ facid = LOG_LOCAL5;
+ } else if( strcasecmp(fac, "local6") == 0 ) {
+ facid = LOG_LOCAL6;
+ } else if( strcasecmp(fac, "local7") == 0 ) {
+ facid = LOG_LOCAL7;
+ } else if( strcasecmp(fac, "user") == 0 ) {
+ facid = LOG_USER;
+ }
+ logctx->logtype = ltSYSLOG;
+ openlog("rteval_parserd", LOG_PID, facid);
+ } else if( strcmp(logdest, "stderr:") == 0 ) {
+ logctx->logtype = ltCONSOLE;
+ logctx->logfp = stderr;
+ } else if( strcmp(logdest, "stdout:") == 0 ) {
+ logctx->logtype = ltCONSOLE;
+ logctx->logfp = stdout;
+ } else {
+ logctx->logtype = ltFILE;
+ logctx->logfp = fopen(logdest, "a");
+ if( logctx->logfp == NULL ) {
+ fprintf(stderr, "** ERROR ** Failed to open log file %s: %s\n",
+ logdest, strerror(errno));
+ free_nullsafe(logctx);
+ return NULL;
+ }
+ }
+ }
+
+ if( logctx->logtype != ltSYSLOG ) {
+ static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
+ logctx->mtx_log = &mtx;
+ }
+ return logctx;
+}
+
+
+/**
+ * Tears down a log context. Closes log files and releases memory used by the log context.
+ *
+ * @param lctx Log context to close
+ */
+void close_log(LogContext *lctx) {
+ if( !lctx ) {
+ return;
+ }
+
+ switch( lctx->logtype ) {
+ case ltFILE:
+ fclose(lctx->logfp);
+ break;
+
+ case ltSYSLOG:
+ closelog();
+ break;
+
+ case ltCONSOLE:
+ break;
+ }
+ free_nullsafe(lctx);
+}
+
+
+/**
+ * Write data to the log.
+ *
+ * @param lctx Log context, where the data will be logged
+ * @param loglvl Log level. See the priorities for syslog(3) for valid values.
+ * @param fmt Data to be logged (stdarg)
+ */
+void writelog(LogContext *lctx, unsigned int loglvl, const char *fmt, ... ) {
+ if( !lctx || !fmt ) {
+ return;
+ }
+
+ if( lctx->verbosity >= loglvl ) {
+ va_list ap;
+
+ va_start(ap, fmt);
+ switch( lctx->logtype ) {
+ case ltSYSLOG:
+ vsyslog(loglvl, fmt, ap);
+ break;
+
+ case ltCONSOLE:
+ case ltFILE:
+ pthread_mutex_lock(lctx->mtx_log);
+ switch( loglvl ) {
+ case LOG_EMERG:
+ fprintf(lctx->logfp, "** EMERG ERROR ** ");
+ break;
+ case LOG_ALERT:
+ fprintf(lctx->logfp, "** ALERT ERROR ** ");
+ break;
+ case LOG_CRIT:
+ fprintf(lctx->logfp, "** CRITICAL ERROR ** ");
+ break;
+ case LOG_ERR:
+ fprintf(lctx->logfp, "** ERROR ** ");
+ break;
+ case LOG_WARNING:
+ fprintf(lctx->logfp, "*WARNING* ");
+ break;
+ case LOG_NOTICE:
+ fprintf(lctx->logfp, "[NOTICE] ");
+ break;
+ case LOG_INFO:
+ fprintf(lctx->logfp, "[INFO] ");
+ break;
+ case LOG_DEBUG:
+ fprintf(lctx->logfp, "[DEBUG] ");
+ break;
+ }
+ vfprintf(lctx->logfp, fmt, ap);
+ fprintf(lctx->logfp, "\n");
+ pthread_mutex_unlock(lctx->mtx_log);
+
+ if( lctx->logtype == ltFILE ) {
+ fflush(lctx->logfp);
+ }
+ break;
+ }
+ va_end(ap);
+ }
+}
diff --git a/server/parser/log.h b/server/parser/log.h
new file mode 100644
index 0000000..1dda0a6
--- /dev/null
+++ b/server/parser/log.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file log.h
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Wed Oct 21 11:38:51 2009
+ *
+ * @brief Generic log functions
+ *
+ */
+
+#ifndef _RTEVAL_LOG_H
+#define _RTEVAL_LOG_H
+
+#include <pthread.h>
+#include <syslog.h>
+
+/**
+ * Supported log types
+ */
+typedef enum { ltSYSLOG, ltFILE, ltCONSOLE } LogType;
+
+/**
+ * The log context structure. Keeps needed information for
+ * a flawless logging experience :-P
+ */
+typedef struct {
+ LogType logtype; /**< What kind of log "device" will be used */
+ FILE *logfp; /**< Only used if logging to stderr, stdout or a file */
+ unsigned int verbosity; /**< Defines which log level the user wants to log */
+ pthread_mutex_t *mtx_log; /**< Mutex to threads to write to a file based log in parallel */
+} LogContext;
+
+
+LogContext *init_log(const char *fname, const char *loglvl);
+void close_log(LogContext *lctx);
+void writelog(LogContext *lctx, unsigned int loglvl, const char *fmt, ... );
+
+#endif
diff --git a/server/parser/parsethread.c b/server/parser/parsethread.c
new file mode 100644
index 0000000..fbf6777
--- /dev/null
+++ b/server/parser/parsethread.c
@@ -0,0 +1,337 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file parsethread.c
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Thu Oct 15 11:52:10 2009
+ *
+ * @brief Contains the "main" function which a parser threads runs
+ *
+ *
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <signal.h>
+#include <pthread.h>
+#include <libgen.h>
+#include <errno.h>
+#include <assert.h>
+
+#include <eurephia_nullsafe.h>
+#include <parsethread.h>
+#include <pgsql.h>
+#include <log.h>
+#include <threadinfo.h>
+#include <statuses.h>
+
+
+/**
+ * Does the same job as 'mkdir -p', but it expects a complete filename as input, and it will
+ * extract the directory path from that filename
+ *
+ * @param fname Full filename containing the directory the report will reside.
+ *
+ * @return Returns 1 on success, otherwise -1
+ */
+static int make_report_dir(LogContext *log, const char *fname) {
+ char *fname_cp = NULL, *dname = NULL, *chkdir = NULL;
+ char *tok = NULL, *saveptr = NULL;
+ int ret = 0;
+ struct stat info;
+
+ if( !fname ) {
+ return 0;
+ }
+
+ fname_cp = strdup(fname);
+ assert( fname_cp != NULL );
+ dname = dirname(fname_cp);
+ chkdir = malloc_nullsafe(log, strlen(dname)+8);
+
+ if( dname[0] == '/' ) {
+ chkdir[0] = '/';
+ }
+
+ // Traverse the directory path, and make sure the directory exists
+ tok = strtok_r(dname, "/", &saveptr);
+ while( tok ) {
+ strcat(chkdir, tok);
+ strcat(chkdir, "/");
+
+ errno = 0;
+ // Check if directory exists
+ if( (stat(chkdir, &info) < 0) ) {
+ switch( errno ) {
+ case ENOENT: // If the directory do not exist, create it
+ if( mkdir(chkdir, 0755) < 0 ) {
+ // If creating dir failed, report error
+ writelog(log, LOG_ALERT,
+ "Could not create directory: %s (%s)",
+ chkdir, strerror(errno));
+ ret = -1;
+ goto exit;
+ }
+ break;
+ default: // If other failure, report that and exit
+ writelog(log, LOG_ALERT,
+ "Could not access directory: %s (%s)",
+ chkdir, strerror(errno));
+ ret = -1;
+ goto exit;
+ }
+ }
+ // Goto next path element
+ tok = strtok_r(NULL, "/", &saveptr);
+ }
+ ret = 1;
+ exit:
+ free_nullsafe(fname_cp);
+ free_nullsafe(chkdir);
+
+ return ret;
+}
+
+
+/**
+ * Builds up a proper full path of where to save the report.
+ *
+ * @param destdir Destination directory for all reports
+ * @param fname Report filename, containing hostname of the reporter
+ * @param rterid rteval run ID
+ *
+ * @return Returns a pointer to a string with the new full path filename on success, otherwise NULL.
+ */
+static char *get_destination_path(LogContext *log, const char *destdir,
+ parseJob_t *job, const int rterid)
+{
+ char *newfname = NULL;
+ int retlen = 0;
+
+ if( !job || rterid < 0 ) {
+ return NULL;
+ }
+
+ retlen = strlen_nullsafe(job->clientid) + strlen(destdir) + 24;
+ newfname = malloc_nullsafe(log, retlen+2);
+
+ snprintf(newfname, retlen, "%s/%s/report-%i.xml", destdir, job->clientid, rterid);
+
+ return newfname;
+}
+
+
+/**
+ * The core parse function. Parses an XML file and stores it in the database according to
+ * the xmlparser.xsl template.
+ *
+ * @param dbc Database connection
+ * @param xslt Pointer to a parsed XSLT Stylesheet (xmlparser.xsl)
+ * @param mtx_sysreg Mutex locking to avoid simultaneous registration of systems, as they cannot
+ * be in an SQL transaction (due to SHA1 sysid must be registered and visible ASAP)
+ * @param destdir Destination directory for the report file, when moved from the queue.
+ * @param job Pointer to a parseJob_t structure containing the job information
+ *
+ * @return Return values:
+ * @code
+ * STAT_SUCCESS : Successfully registered report
+ * STAT_XMLFAIL : Could not parse the XML report file
+ * STAT_SYSREG : Failed to register the system into the systems or systems_hostname tables
+ * STAT_RTERIDREG: Failed to get a new rterid value
+ * STAT_GENDB : Failed to start an SQL transaction (BEGIN)
+ * STAT_RTEVRUNS : Failed to register the rteval run into rtevalruns or rtevalruns_details
+ * STAT_CYCLIC : Failed to register the data into cyclic_statistics or cyclic_rawdata tables
+ * STAT_REPMOVE : Failed to move the report file
+ * @endcode
+ */
+inline int parse_report(dbconn *dbc, xsltStylesheet *xslt, pthread_mutex_t *mtx_sysreg,
+ const char *destdir, parseJob_t *job) {
+ int syskey = -1, rterid = -1;
+ int rc = -1;
+ xmlDoc *repxml = NULL;
+ char *destfname;
+
+ repxml = xmlParseFile(job->filename);
+ if( !repxml ) {
+ writelog(dbc->log, LOG_ERR,
+ "Could not parse XML file: %s", job->filename);
+ return STAT_XMLFAIL;
+ }
+
+ pthread_mutex_lock(mtx_sysreg);
+ syskey = db_register_system(dbc, xslt, repxml);
+ if( syskey < 0 ) {
+ writelog(dbc->log, LOG_ERR,
+ "Failed to register system (XML file: %s)", job->filename);
+ rc = STAT_SYSREG;
+ goto exit;
+
+ }
+ rterid = db_get_new_rterid(dbc);
+ if( rterid < 0 ) {
+ writelog(dbc->log, LOG_ERR,
+ "Failed to register rteval run (XML file: %s)", job->filename);
+ rc = STAT_RTERIDREG;
+ goto exit;
+ }
+ pthread_mutex_unlock(mtx_sysreg);
+
+ if( db_begin(dbc) < 1 ) {
+ rc = STAT_GENDB;
+ goto exit;
+ }
+
+ // Create a new filename of where to save the report
+ destfname = get_destination_path(dbc->log, destdir, job, rterid);
+ if( !destfname ) {
+ writelog(dbc->log, LOG_ERR,
+ "Failed to generate local report filename for (%i) %s",
+ job->submid, job->filename);
+ db_rollback(dbc);
+ rc = STAT_UNKNFAIL;
+ goto exit;
+ }
+
+ if( db_register_rtevalrun(dbc, xslt, repxml, job->submid, syskey, rterid, destfname) < 0 ) {
+ writelog(dbc->log, LOG_ERR,
+ "Failed to register rteval run (XML file: %s)",
+ job->filename);
+ db_rollback(dbc);
+ rc = STAT_RTEVRUNS;
+ goto exit;
+ }
+
+ if( db_register_cyclictest(dbc, xslt, repxml, rterid) != 1 ) {
+ writelog(dbc->log, LOG_ERR,
+ "Failed to register cyclictest data (XML file: %s)",
+ job->filename);
+ db_rollback(dbc);
+ rc = STAT_CYCLIC;
+ goto exit;
+ }
+
+ // When all database registrations are done, move the file to it's right place
+ if( make_report_dir(dbc->log, destfname) < 1 ) { // Make sure report directory exists
+ db_rollback(dbc);
+ rc = STAT_REPMOVE;
+ goto exit;
+ }
+
+ if( rename(job->filename, destfname) < 0 ) { // Move the file
+ writelog(dbc->log, LOG_ERR,
+ "Failed to move report file from %s to %s (%s)",
+ job->filename, destfname, strerror(errno));
+ db_rollback(dbc);
+ rc = STAT_REPMOVE;
+ goto exit;
+ }
+ free_nullsafe(destfname);
+
+ rc = STAT_SUCCESS;
+ db_commit(dbc);
+
+ exit:
+ xmlFreeDoc(repxml);
+ return rc;
+}
+
+
+/**
+ * The parser thread. This thread lives until a shutdown notification is received. It pulls
+ * messages on a POSIX MQ based message queue containing submission ID and full path to an XML
+ * report to be parsed.
+ *
+ * @param thrargs Contains database connection, XSLT stylesheet, POSXI MQ descriptor, etc
+ *
+ * @return Returns 0 on successful operation, otherwise 1 on errors.
+ */
+void *parsethread(void *thrargs) {
+ threadData_t *args = (threadData_t *) thrargs;
+ parseJob_t jobinfo;
+ long exitcode = 0;
+
+ writelog(args->dbc->log, LOG_DEBUG, "[Thread %i] Starting", args->id);
+ pthread_mutex_lock(args->mtx_thrcnt);
+ (*(args->threadcount)) += 1;
+ pthread_mutex_unlock(args->mtx_thrcnt);
+
+ // Polling loop
+ while( *(args->shutdown) == 0 ) {
+ int len = 0;
+ unsigned int prio = 0;
+
+ // Check if the database connection is alive before pulling any messages
+ if( db_ping(args->dbc) != 1 ) {
+ writelog(args->dbc->log, LOG_EMERG,
+ "[Thread %i] Lost database conneciting: Shutting down thread.",
+ args->id);
+
+ if( *(args->threadcount) <= 1 ) {
+ writelog(args->dbc->log, LOG_EMERG,
+ "No more worker threads available. "
+ "Initiating complete shutdown");
+ kill(getpid(), SIGUSR1);
+ }
+ exitcode = 1;
+ goto exit;
+ }
+
+ // Retrieve a parse job from the message queue
+ memset(&jobinfo, 0, sizeof(parseJob_t));
+ errno = 0;
+ len = mq_receive(args->msgq, (char *)&jobinfo, sizeof(parseJob_t), &prio);
+ if( (len < 0) && errno != EAGAIN ) {
+ writelog(args->dbc->log, LOG_CRIT,
+ "Could not receive the message from queue: %s",
+ strerror(errno));
+ pthread_exit((void *) 1);
+ }
+
+ // Ignore whatever message if the shutdown flag is set.
+ if( *(args->shutdown) != 0 ) {
+ break;
+ }
+
+ // If we have a message, then process the parse job
+ if( (errno != EAGAIN) && (len > 0) ) {
+ int res = 0;
+
+ writelog(args->dbc->log, LOG_DEBUG,
+ "** Thread %i: Job recieved, submid: %i",
+ args->id, jobinfo.submid);
+
+ // Mark the job as "in progress", if successful update, continue parsing it
+ if( db_update_submissionqueue(args->dbc, jobinfo.submid, STAT_INPROG) ) {
+ res = parse_report(args->dbc, args->xslt, args->mtx_sysreg,
+ args->destdir, &jobinfo);
+ // Set the status for the submission
+ db_update_submissionqueue(args->dbc, jobinfo.submid, res);
+ } else {
+ writelog(args->dbc->log, LOG_CRIT,
+ "Failed to mark submid %i as STAT_INPROG",
+ jobinfo.submid);
+ }
+ }
+ }
+ writelog(args->dbc->log, LOG_DEBUG, "[Thread %i] Shut down", args->id);
+ exit:
+ pthread_mutex_lock(args->mtx_thrcnt);
+ (*(args->threadcount)) -= 1;
+ pthread_mutex_unlock(args->mtx_thrcnt);
+
+ pthread_exit((void *) exitcode);
+}
diff --git a/server/parser/parsethread.h b/server/parser/parsethread.h
new file mode 100644
index 0000000..66e8524
--- /dev/null
+++ b/server/parser/parsethread.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file parsethread.h
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Thu Oct 15 11:52:10 2009
+ *
+ * @brief Contains the "main" function which a parser threads runs
+ *
+ */
+
+#ifndef _PARSETHREAD_H
+#define _PARSETHREAD_H
+
+/**
+ * jbNONE means no job available,
+ * jbAVAIL indicates that parseJob_t contains a job
+*/
+typedef enum { jbNONE, jbAVAIL } jobStatus;
+
+/**
+ * This struct is used for sending a parse job to a worker thread via POSIX MQ
+ */
+typedef struct {
+ jobStatus status; /**< Info about if job information*/
+ unsigned int submid; /**< Work info: Numeric ID of the job being parsed */
+ char clientid[256]; /**< Work info: Should contain senders hostname */
+ char filename[4096]; /**< Work info: Full filename of the report to be parsed */
+} parseJob_t;
+
+
+void *parsethread(void *thrargs);
+
+#endif
diff --git a/server/parser/pgsql.c b/server/parser/pgsql.c
new file mode 100644
index 0000000..7a6e7ec
--- /dev/null
+++ b/server/parser/pgsql.c
@@ -0,0 +1,989 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file pgsql.c
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Wed Oct 13 17:44:35 2009
+ *
+ * @brief Database API for the PostgreSQL database.
+ *
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+#include <assert.h>
+#include <errno.h>
+
+#include <libpq-fe.h>
+
+#include <libxml/parser.h>
+#include <libxml/xmlsave.h>
+#include <libxslt/transform.h>
+#include <libxslt/xsltutils.h>
+
+#include <eurephia_nullsafe.h>
+#include <eurephia_xml.h>
+#include <eurephia_values.h>
+#include <configparser.h>
+#include <xmlparser.h>
+#include <pgsql.h>
+#include <log.h>
+#include <statuses.h>
+
+/**
+ * Connect to a database, based on the given configuration
+ *
+ * @param cfg eurephiaVALUES containing the configuration
+ * @param id Database connection ID. Used to identify which thread is doing what with the database
+ * @param log Log context, where all logging will go
+ *
+ * @return Returns a database connection context
+ */
+dbconn *db_connect(eurephiaVALUES *cfg, unsigned int id, LogContext *log) {
+ dbconn *ret = NULL;
+
+ ret = (dbconn *) malloc_nullsafe(log, sizeof(dbconn)+2);
+ ret->id = id;
+ ret->log = log;
+
+ writelog(log, LOG_DEBUG, "[Connection %i] Connecting to database: server=%s:%s, "
+ "database=%s, user=%s", ret->id,
+ eGet_value(cfg, "db_server"), eGet_value(cfg, "db_port"),
+ eGet_value(cfg, "database"), eGet_value(cfg, "db_username"));
+ ret->db = PQsetdbLogin(eGet_value(cfg, "db_server"),
+ eGet_value(cfg, "db_port"),
+ NULL, /* pgopt */
+ NULL, /* pgtty */
+ eGet_value(cfg, "database"),
+ eGet_value(cfg, "db_username"),
+ eGet_value(cfg, "db_password"));
+
+ if( !ret->db ) {
+ writelog(log, LOG_EMERG,
+ "[Connection %i] Could not connect to the database (unknown reason)", ret->id);
+ free_nullsafe(ret);
+ return NULL;
+ }
+
+ if( PQstatus(ret->db) != CONNECTION_OK ) {
+ writelog(log, LOG_EMERG, "[Connection %i] Failed to connect to the database: %s",
+ ret->id, PQerrorMessage(ret->db));
+ free_nullsafe(ret);
+ return NULL;
+ }
+ return ret;
+}
+
+
+/**
+ * Pings the database connection to check if it is alive
+ *
+ * @param dbc Database connection to ping
+ *
+ * @return Returns 1 if the connection is alive, otherwise 0
+ */
+int db_ping(dbconn *dbc) {
+ PGresult *res = NULL;
+
+ // Send ping
+ res = PQexec(dbc->db, "");
+ PQclear(res);
+
+ // Check status
+ if( PQstatus(dbc->db) != CONNECTION_OK ) {
+ PQreset(dbc->db);
+ if( PQstatus(dbc->db) != CONNECTION_OK ) {
+ writelog(dbc->log, LOG_EMERG,
+ "[Connection %i] Database error - Lost connection: %s",
+ dbc->id, PQerrorMessage(dbc->db));
+ return 0;
+ } else {
+ writelog(dbc->log, LOG_CRIT,
+ "[Conncetion %i] Database connection restored", dbc->id);
+ }
+ }
+ return 1;
+}
+
+
+/**
+ * Disconnect from the database
+ *
+ * @param dbc Pointer to the database handle to be disconnected.
+ */
+void db_disconnect(dbconn *dbc) {
+ if( dbc && dbc->db ) {
+ writelog(dbc->log, LOG_DEBUG, "[Connection %i] Disconnecting from database", dbc->id);
+ PQfinish(dbc->db);
+ dbc->db = NULL;
+ dbc->log = NULL;
+ }
+ free_nullsafe(dbc);
+}
+
+
+/**
+ * This function does INSERT SQL queries based on an XML document (sqldata) which contains
+ * all information about table, fields and records to be inserted. For security and performance,
+ * this function uses prepared SQL statements.
+ *
+ * This function is PostgreSQL specific.
+ *
+ * @param dbc Database handler to a PostgreSQL
+ * @param sqldoc sqldata XML document containing the data to be inserted.
+ *
+ * The sqldata XML document must be formated like this:
+ * @code
+ * <sqldata table="{table name}" [key="{field name}">
+ * <fields>
+ * <field fid="{integer}">{field name}</field>
+ * ...
+ * ...
+ * <field fid="{integer_n}">{field name 'n'}</field>
+ * </fields>
+ * <records>
+ * <record>
+ * <value fid="{integer} [type="{data type}"] [hash="{hash type}">{value for field 'fid'</value>
+ * ...
+ * ...
+ * <value fid="{integer_n}">{value for field 'fid_n'</value>
+ * </record>
+ * ...
+ * ...
+ * ...
+ * </records>
+ * </sqldata>
+ * @endcode
+ * The 'sqldata' root tag must contain a 'table' attribute. This must contain the a name of a table
+ * in the database. If the 'key' attribute is set, the function will return the that field value for
+ * each INSERT query, using INSERT ... RETURNING {field name}. The sqldata root tag must then have
+ * two children, 'fields' and 'records'.
+ *
+ * The 'fields' tag need to contain 'field' children tags for each field to insert data for. Each
+ * field in the fields tag must be assigned a unique integer.
+ *
+ * The 'records' tag need to contain 'record' children tags for each record to be inserted. Each
+ * record tag needs to have 'value' tags for each field which is found in the 'fields' section.
+ *
+ * The 'value' tags must have a 'fid' attribute. This is the link between the field name in the
+ * 'fields' section and the value to be inserted.
+ *
+ * The 'type' attribute may be used as well, but the only supported data type supported to this
+ * attribute is 'xmlblob'. In this case, the contents of the 'value' tag must be more XML tags.
+ * These tags will then be serialised to a string which is inserted into the database.
+ *
+ * The 'hash' attribute of the 'value' tag can be set to 'sha1'. This will make do a SHA1 hash
+ * calculation of the value and this hash value will be used for the insert.
+ *
+ * @return Returns an eurephiaVALUES list containing information about each record which was inserted.
+ * If the 'key' attribute is not set in the 'sqldata' tag, the OID value of each record will be
+ * saved. If the table do not support OIDs, the value will be '0'. Otherwise the contents of
+ * the defined field name will be returned. If one of the INSERT queries fails, it will abort
+ * further processing and the function will return NULL.
+ */
+eurephiaVALUES *pgsql_INSERT(dbconn *dbc, xmlDoc *sqldoc) {
+ xmlNode *root_n = NULL, *fields_n = NULL, *recs_n = NULL, *ptr_n = NULL, *val_n = NULL;
+ char **field_ar = NULL, *fields = NULL, **value_ar = NULL, *values = NULL, *table = NULL,
+ tmp[20], *sql = NULL, *key = NULL;
+ unsigned int fieldcnt = 0, *field_idx, i = 0;
+ PGresult *dbres = NULL;
+ eurephiaVALUES *res = NULL;
+
+ assert( (dbc != NULL) && (sqldoc != NULL) );
+
+ root_n = xmlDocGetRootElement(sqldoc);
+ if( !root_n || (xmlStrcmp(root_n->name, (xmlChar *) "sqldata") != 0) ) {
+ writelog(dbc->log, LOG_ERR,
+ "[Connection %i] Input XML document is not a valid sqldata document", dbc->id);
+ return NULL;
+ }
+
+ table = xmlGetAttrValue(root_n->properties, "table");
+ if( !table ) {
+ writelog(dbc->log, LOG_ERR,
+ "[Connection %i] Input XML document is missing table reference", dbc->id);
+ return NULL;
+ }
+
+ key = xmlGetAttrValue(root_n->properties, "key");
+
+ fields_n = xmlFindNode(root_n, "fields");
+ recs_n = xmlFindNode(root_n, "records");
+ if( !fields_n || !recs_n ) {
+ writelog(dbc->log, LOG_ERR,
+ "[Connection %i] Input XML document is missing either <fields/> or <records/>",
+ dbc->id);
+ return NULL;
+ }
+
+ // Count number of fields
+ foreach_xmlnode(fields_n->children, ptr_n) {
+ if( ptr_n->type == XML_ELEMENT_NODE ) {
+ fieldcnt++;
+ }
+ }
+
+ // Generate lists of all fields and a index mapping table
+ field_idx = calloc(fieldcnt+1, sizeof(unsigned int));
+ field_ar = calloc(fieldcnt+1, sizeof(char *));
+ foreach_xmlnode(fields_n->children, ptr_n) {
+ if( ptr_n->type != XML_ELEMENT_NODE ) {
+ continue;
+ }
+
+ field_idx[i] = atoi_nullsafe(xmlGetAttrValue(ptr_n->properties, "fid"));
+ field_ar[i] = xmlExtractContent(ptr_n);
+ i++;
+ }
+
+ // Generate strings with field names and value place holders
+ // for a prepared SQL statement
+ fields = malloc_nullsafe(dbc->log, 3);
+ values = malloc_nullsafe(dbc->log, 6*(fieldcnt+1));
+ strcpy(fields, "(");
+ strcpy(values, "(");
+ int len = 3;
+ for( i = 0; i < fieldcnt; i++ ) {
+ // Prepare VALUES section
+ snprintf(tmp, 6, "$%i", i+1);
+ append_str(values, tmp, (6*fieldcnt));
+
+ // Prepare fields section
+ len += strlen_nullsafe(field_ar[i])+2;
+ fields = realloc(fields, len);
+ strcat(fields, field_ar[i]);
+
+ if( i < (fieldcnt-1) ) {
+ strcat(fields, ",");
+ strcat(values, ",");
+ }
+ }
+ strcat(fields, ")");
+ strcat(values, ")");
+
+ // Build up the SQL query
+ sql = malloc_nullsafe(dbc->log,
+ strlen_nullsafe(fields)
+ + strlen_nullsafe(values)
+ + strlen_nullsafe(table)
+ + strlen_nullsafe(key)
+ + 34 /* INSERT INTO VALUES RETURNING*/
+ );
+ sprintf(sql, "INSERT INTO %s %s VALUES %s", table, fields, values);
+ if( key ) {
+ strcat(sql, " RETURNING ");
+ strcat(sql, key);
+ }
+
+ // Create a prepared SQL query
+#ifdef DEBUG_SQL
+ writelog(dbc->log, LOG_DEBUG, "[Connection %i] Preparing SQL statement: %s", dbc->id, sql);
+#endif
+ dbres = PQprepare(dbc->db, "", sql, fieldcnt, NULL);
+ if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) {
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Failed to prepare SQL query: %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ PQclear(dbres);
+ goto exit;
+ }
+ PQclear(dbres);
+
+ // Loop through all records and generate SQL statements
+ res = eCreate_value_space(dbc->log, 1);
+ foreach_xmlnode(recs_n->children, ptr_n) {
+ if( ptr_n->type != XML_ELEMENT_NODE ) {
+ continue;
+ }
+
+ // Loop through all value nodes in each record node and get the values for each field
+ value_ar = calloc(fieldcnt, sizeof(char *));
+ i = 0;
+ foreach_xmlnode(ptr_n->children, val_n) {
+ char *fid_s = NULL;
+ int fid = -1;
+
+ if( i > fieldcnt ) {
+ break;
+ }
+
+ if( val_n->type != XML_ELEMENT_NODE ) {
+ continue;
+ }
+
+ fid_s = xmlGetAttrValue(val_n->properties, "fid");
+ fid = atoi_nullsafe(fid_s);
+ if( (fid_s == NULL) || (fid < 0) ) {
+ continue;
+ }
+ value_ar[field_idx[i]] = sqldataExtractContent(dbc->log, val_n);
+ i++;
+ }
+
+ // Insert the record into the database
+ dbres = PQexecPrepared(dbc->db, "", fieldcnt,
+ (const char * const *)value_ar, NULL, NULL, 0);
+ if( PQresultStatus(dbres) != (key ? PGRES_TUPLES_OK : PGRES_COMMAND_OK) ) {
+ writelog(dbc->log, LOG_ALERT, "[Connection %i] Failed to do SQL INSERT query: %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ PQclear(dbres);
+ eFree_values(res);
+ res = NULL;
+
+ // Free up the memory we've used for this record
+ for( i = 0; i < fieldcnt; i++ ) {
+ free_nullsafe(value_ar[i]);
+ }
+ free_nullsafe(value_ar);
+ goto exit;
+ }
+ if( key ) {
+ // If the /sqldata/@key attribute was set, fetch the returning ID
+ eAdd_value(res, key, PQgetvalue(dbres, 0, 0));
+ } else {
+ static char oid[32];
+ snprintf(oid, 30, "%ld%c", (unsigned long int) PQoidValue(dbres), 0);
+ eAdd_value(res, "oid", oid);
+ }
+ PQclear(dbres);
+
+ // Free up the memory we've used for this record
+ for( i = 0; i < fieldcnt; i++ ) {
+ free_nullsafe(value_ar[i]);
+ }
+ free_nullsafe(value_ar);
+ }
+
+ exit:
+ free_nullsafe(sql);
+ free_nullsafe(fields);
+ free_nullsafe(values);
+ free_nullsafe(field_ar);
+ free_nullsafe(field_idx);
+ return res;
+}
+
+
+/**
+ * Start an SQL transaction (SQL BEGIN)
+ *
+ * @param dbc Database handler where to perform the SQL queries
+ *
+ * @return Returns 1 on success, otherwise -1 is returned
+ */
+int db_begin(dbconn *dbc) {
+ PGresult *dbres = NULL;
+
+ dbres = PQexec(dbc->db, "BEGIN");
+ if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) {
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Failed to do prepare a transaction (BEGIN): %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ PQclear(dbres);
+ return -1;
+ }
+ PQclear(dbres);
+ return 1;
+}
+
+
+/**
+ * Commits an SQL transaction (SQL COMMIT)
+ *
+ * @param dbc Database handler where to perform the SQL queries
+ *
+ * @return Returns 1 on success, otherwise -1 is returned
+ */
+int db_commit(dbconn *dbc) {
+ PGresult *dbres = NULL;
+
+ dbres = PQexec(dbc->db, "COMMIT");
+ if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) {
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Failed to do commit a database transaction (COMMIT): %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ PQclear(dbres);
+ return -1;
+ }
+ PQclear(dbres);
+ return 1;
+}
+
+
+/**
+ * Aborts an SQL transaction (SQL ROLLBACK/ABORT)
+ *
+ * @param dbc Database handler where to perform the SQL queries
+ *
+ * @return Returns 1 on success, otherwise -1 is returned
+ */
+int db_rollback(dbconn *dbc) {
+ PGresult *dbres = NULL;
+
+ dbres = PQexec(dbc->db, "ROLLBACK");
+ if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) {
+ writelog(dbc->log, LOG_CRIT,
+ "[Connection %i] Failed to do abort/rollback a transaction (ROLLBACK): %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ PQclear(dbres);
+ return -1;
+ }
+ PQclear(dbres);
+ return 1;
+}
+
+
+/**
+ * This function blocks until a notification is received from the database
+ *
+ * @param dbc Database connection
+ * @param shutdown Pointer to the shutdown flag. Used to avoid reporting false errors.
+ * @param listenfor Name to be used when calling LISTEN
+ *
+ * @return Returns 1 on successful waiting, otherwise -1
+ */
+int db_wait_notification(dbconn *dbc, const int *shutdown, const char *listenfor) {
+ int sock, ret = 0;
+ PGresult *dbres = NULL;
+ PGnotify *notify = NULL;
+ fd_set input_mask;
+ char *sql = NULL;
+
+ sql = malloc_nullsafe(dbc->log, strlen_nullsafe(listenfor) + 12);
+ assert( sql != NULL );
+
+ // Initiate listening
+ sprintf(sql, "LISTEN %s", listenfor);
+ dbres = PQexec(dbc->db, sql);
+ if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) {
+ writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ free_nullsafe(sql);
+ PQclear(dbres);
+ return -1;
+ }
+ PQclear(dbres);
+
+ // Start listening and waiting
+ while( ret == 0 ) {
+ sock = PQsocket(dbc->db);
+ if (sock < 0) {
+ // shouldn't happen
+ ret = -1;
+ break;
+ }
+
+ // Wait for something to happen on the database socket
+ FD_ZERO(&input_mask);
+ FD_SET(sock, &input_mask);
+ if (select(sock + 1, &input_mask, NULL, NULL, NULL) < 0) {
+ // If the shutdown flag is set, select() will fail due to a signal. Only
+ // report errors if we're not shutting down, or else exit normally with
+ // successful waiting.
+ if( *shutdown == 0 ) {
+ writelog(dbc->log, LOG_CRIT, "[Connection %i] select() failed: %s",
+ dbc->id, strerror(errno));
+ ret = -1;
+ goto exit;
+ } else {
+ ret = 1;
+ }
+ break;
+ }
+
+ // Process the event
+ PQconsumeInput(dbc->db);
+
+ // Check if connection still is valid
+ if( PQstatus(dbc->db) != CONNECTION_OK ) {
+ PQreset(dbc->db);
+ if( PQstatus(dbc->db) != CONNECTION_OK ) {
+ writelog(dbc->log, LOG_EMERG,
+ "[Connection %i] Database connection died: %s",
+ dbc->id, PQerrorMessage(dbc->db));
+ ret = -1;
+ goto exit;
+ }
+ writelog(dbc->log, LOG_CRIT,
+ "[Connection %i] Database connection restored", dbc->id);
+ }
+
+ while ((notify = PQnotifies(dbc->db)) != NULL) {
+ // If a notification was received, inform and exit with success.
+ writelog(dbc->log, LOG_DEBUG,
+ "[Connection %i] Received notfication from pid %d",
+ dbc->id, notify->be_pid);
+ PQfreemem(notify);
+ ret = 1;
+ break;
+ }
+ }
+
+ // Stop listening when we exit
+ sprintf(sql, "UNLISTEN %s", listenfor);
+ dbres = PQexec(dbc->db, sql);
+ if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) {
+ writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ free_nullsafe(sql);
+ ret = -1;
+ }
+ free_nullsafe(sql);
+ PQclear(dbres);
+
+ exit:
+ return ret;
+}
+
+
+/**
+ * Retrive the first available submitted report
+ *
+ * @param dbc Database connection
+ * @param mtx pthread_mutex to avoid parallel access to the submission queue table, to avoid
+ * the same job being retrieved multiple times.
+ *
+ * @return Returns a pointer to a parseJob_t struct, with the parse job info on success, otherwise NULL
+ */
+parseJob_t *db_get_submissionqueue_job(dbconn *dbc, pthread_mutex_t *mtx) {
+ parseJob_t *job = NULL;
+ PGresult *res = NULL;
+ char sql[4098];
+
+ job = (parseJob_t *) malloc_nullsafe(dbc->log, sizeof(parseJob_t));
+
+ // Get the first available submission
+ memset(&sql, 0, 4098);
+ snprintf(sql, 4096,
+ "SELECT submid, filename, clientid"
+ " FROM submissionqueue"
+ " WHERE status = %i"
+ " ORDER BY submid"
+ " LIMIT 1",
+ STAT_NEW);
+
+ pthread_mutex_lock(mtx);
+ res = PQexec(dbc->db, sql);
+ if( PQresultStatus(res) != PGRES_TUPLES_OK ) {
+ pthread_mutex_unlock(mtx);
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Failed to query submission queue (SELECT): %s",
+ dbc->id, PQresultErrorMessage(res));
+ PQclear(res);
+ free_nullsafe(job);
+ return NULL;
+ }
+
+ if( PQntuples(res) == 1 ) {
+ job->status = jbAVAIL;
+ job->submid = atoi_nullsafe(PQgetvalue(res, 0, 0));
+ snprintf(job->filename, 4095, "%.4094s", PQgetvalue(res, 0, 1));
+ snprintf(job->clientid, 255, "%.254s", PQgetvalue(res, 0, 2));
+
+ // Update the submission queue status
+ if( db_update_submissionqueue(dbc, job->submid, STAT_ASSIGNED) < 1 ) {
+ pthread_mutex_unlock(mtx);
+ writelog(dbc->log, LOG_ALERT, "[Connection %i] Failed to update "
+ "submission queue statis to STAT_ASSIGNED", dbc->id);
+ free_nullsafe(job);
+ return NULL;
+ }
+ } else {
+ job->status = jbNONE;
+ }
+ pthread_mutex_unlock(mtx);
+ PQclear(res);
+ return job;
+}
+
+
+/**
+ * Updates the submission queue table with the new status and the appropriate timestamps
+ *
+ * @param dbc Database handler to the rteval database
+ * @param submid Submission ID to update
+ * @param status The new status
+ *
+ * @return Returns 1 on success, 0 on invalid status ID and -1 on database errors.
+ */
+int db_update_submissionqueue(dbconn *dbc, unsigned int submid, int status) {
+ PGresult *res = NULL;
+ char sql[4098];
+
+ memset(&sql, 0, 4098);
+ switch( status ) {
+ case STAT_ASSIGNED:
+ case STAT_RTERIDREG:
+ case STAT_REPMOVE:
+ snprintf(sql, 4096,
+ "UPDATE submissionqueue SET status = %i"
+ " WHERE submid = %i", status, submid);
+ break;
+
+ case STAT_INPROG:
+ snprintf(sql, 4096,
+ "UPDATE submissionqueue SET status = %i, parsestart = NOW()"
+ " WHERE submid = %i", status, submid);
+ break;
+
+ case STAT_SUCCESS:
+ case STAT_UNKNFAIL:
+ case STAT_XMLFAIL:
+ case STAT_SYSREG:
+ case STAT_GENDB:
+ case STAT_RTEVRUNS:
+ case STAT_CYCLIC:
+ snprintf(sql, 4096,
+ "UPDATE submissionqueue SET status = %i, parseend = NOW() WHERE submid = %i",
+ status, submid);
+ break;
+
+ default:
+ case STAT_NEW:
+ writelog(dbc->log, LOG_ERR,
+ "[Connection %i] Invalid status (%i) attempted to set on submid %i",
+ dbc->id, status, submid);
+ return 0;
+ }
+
+ res = PQexec(dbc->db, sql);
+ if( !res ) {
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Unkown error when updating submid %i to status %i",
+ dbc->id, submid, status);
+ return -1;
+ } else if( PQresultStatus(res) != PGRES_COMMAND_OK ) {
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Failed to UPDATE submissionqueue (submid: %i, status: %i): %s",
+ dbc->id, submid, status, PQresultErrorMessage(res));
+ PQclear(res);
+ return -1;
+ }
+ PQclear(res);
+ return 1;
+}
+
+
+/**
+ * Registers information into the 'systems' and 'systems_hostname' tables, based on the
+ * summary/report XML file from rteval.
+ *
+ * @param dbc Database handler where to perform the SQL queries
+ * @param xslt A pointer to a parsed 'xmlparser.xsl' XSLT template
+ * @param summaryxml The XML report from rteval
+ *
+ * @return Returns a value > 0 on success, which is a unique reference to the system of the report.
+ * If the function detects that this system is already registered, the 'syskey' reference will
+ * be reused. On errors, -1 will be returned.
+ */
+int db_register_system(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml) {
+ PGresult *dbres = NULL;
+ eurephiaVALUES *dbdata = NULL;
+ xmlDoc *sysinfo_d = NULL, *hostinfo_d = NULL;
+ parseParams prms;
+ char sqlq[4098];
+ char *sysid = NULL; // SHA1 value of the system id
+ char *ipaddr = NULL, *hostname = NULL;
+ int syskey = -1;
+
+ memset(&prms, 0, sizeof(parseParams));
+ prms.table = "systems";
+ sysinfo_d = parseToSQLdata(dbc->log, xslt, summaryxml, &prms);
+ if( !sysinfo_d ) {
+ writelog(dbc->log, LOG_ERR, "[Connection %i] Could not parse the input XML data", dbc->id);
+ syskey= -1;
+ goto exit;
+ }
+ sysid = sqldataGetValue(dbc->log, sysinfo_d, "sysid", 0);
+ if( !sysid ) {
+ writelog(dbc->log, LOG_ERR,
+ "[Connection %i] Could not retrieve the sysid field from the input XML", dbc->id);
+ syskey= -1;
+ goto exit;
+ }
+
+ memset(&sqlq, 0, 4098);
+ snprintf(sqlq, 4096, "SELECT syskey FROM systems WHERE sysid = '%.256s'", sysid);
+ free_nullsafe(sysid);
+ dbres = PQexec(dbc->db, sqlq);
+ if( PQresultStatus(dbres) != PGRES_TUPLES_OK ) {
+ writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ writelog(dbc->log, LOG_DEBUG, "[Connection %i] Failing SQL query: %s",
+ dbc->id, sqlq);
+ PQclear(dbres);
+ syskey= -1;
+ goto exit;
+ }
+
+ if( PQntuples(dbres) == 0 ) { // No record found, need to register this system
+ PQclear(dbres);
+
+ dbdata = pgsql_INSERT(dbc, sysinfo_d);
+ if( !dbdata ) {
+ syskey= -1;
+ goto exit;
+ }
+ if( (eCount(dbdata) != 1) || !dbdata->val ) { // Only one record should be registered
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Failed to register the system", dbc->id);
+ eFree_values(dbdata);
+ syskey= -1;
+ goto exit;
+ }
+ syskey = atoi_nullsafe(dbdata->val);
+ hostinfo_d = sqldataGetHostInfo(dbc->log, xslt, summaryxml, syskey, &hostname, &ipaddr);
+ if( !hostinfo_d ) {
+ syskey = -1;
+ goto exit;
+ }
+ eFree_values(dbdata);
+
+ dbdata = pgsql_INSERT(dbc, hostinfo_d);
+ syskey = (dbdata ? syskey : -1);
+ eFree_values(dbdata);
+
+ } else if( PQntuples(dbres) == 1 ) { // System found - check if the host IP is known or not
+ syskey = atoi_nullsafe(PQgetvalue(dbres, 0, 0));
+ hostinfo_d = sqldataGetHostInfo(dbc->log, xslt, summaryxml, syskey, &hostname, &ipaddr);
+ if( !hostinfo_d ) {
+ syskey = -1;
+ goto exit;
+ }
+ PQclear(dbres);
+
+ // Check if this hostname and IP address is registered
+ snprintf(sqlq, 4096,
+ "SELECT syskey FROM systems_hostname"
+ " WHERE hostname='%.256s' AND ipaddr='%.64s'",
+ hostname, ipaddr);
+ dbres = PQexec(dbc->db, sqlq);
+ if( PQresultStatus(dbres) != PGRES_TUPLES_OK ) {
+ writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ writelog(dbc->log, LOG_DEBUG, "[Connection %i] Failing SQL query: %s",
+ dbc->id, sqlq);
+ PQclear(dbres);
+ syskey= -1;
+ goto exit;
+ }
+
+ if( PQntuples(dbres) == 0 ) { // Not registered, then register it
+ dbdata = pgsql_INSERT(dbc, hostinfo_d);
+ syskey = (dbdata ? syskey : -1);
+ eFree_values(dbdata);
+ }
+ PQclear(dbres);
+ } else {
+ // Critical -- system IDs should not be registered more than once
+ writelog(dbc->log, LOG_CRIT, "[Connection %i] Multiple systems registered (%s)",
+ dbc->id, sqlq);
+ syskey= -1;
+ }
+
+ exit:
+ free_nullsafe(hostname);
+ free_nullsafe(ipaddr);
+ if( sysinfo_d ) {
+ xmlFreeDoc(sysinfo_d);
+ }
+ if( hostinfo_d ) {
+ xmlFreeDoc(hostinfo_d);
+ }
+ return syskey;
+}
+
+
+/**
+ * Retrieves the next available rteval run ID (rterid)
+ *
+ * @param dbc Database handler where to perform the SQL query
+ *
+ * @return Returns a value > 0 on success, containing the assigned rterid value. Otherwise -1 is returned.
+ */
+int db_get_new_rterid(dbconn *dbc) {
+ PGresult *dbres = NULL;
+ int rterid = 0;
+
+ dbres = PQexec(dbc->db, "SELECT nextval('rtevalruns_rterid_seq')");
+ if( (PQresultStatus(dbres) != PGRES_TUPLES_OK) || (PQntuples(dbres) != 1) ) {
+ rterid = -1;
+ } else {
+ rterid = atoi_nullsafe(PQgetvalue(dbres, 0, 0));
+ }
+
+ if( rterid < 1 ) {
+ writelog(dbc->log, LOG_CRIT,
+ "[Connection %i] Failed to retrieve a new rterid value", dbc->id);
+ }
+ if( rterid < 0 ) {
+ writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s",
+ dbc->id, PQresultErrorMessage(dbres));
+ }
+ PQclear(dbres);
+ return rterid;
+}
+
+
+/**
+ * Registers information into the 'rtevalruns' and 'rtevalruns_details' tables
+ *
+ * @param dbc Database handler where to perform the SQL queries
+ * @param xslt A pointer to a parsed 'xmlparser.xsl' XSLT template
+ * @param summaryxml The XML report from rteval
+ * @param submid Submission ID, referencing the record in the submissionqueue table.
+ * @param syskey A positive integer containing the return value from db_register_system()
+ * @param rterid A positive integer containing the return value from db_get_new_rterid()
+ * @param report_fname A string containing the filename of the report.
+ *
+ * @return Returns 1 on success, otherwise -1 is returned.
+ */
+int db_register_rtevalrun(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml,
+ unsigned int submid, int syskey, int rterid, const char *report_fname)
+{
+ int ret = -1;
+ xmlDoc *rtevalrun_d = NULL, *rtevalrundets_d = NULL;
+ parseParams prms;
+ eurephiaVALUES *dbdata = NULL;
+
+ // Parse the rtevalruns information
+ memset(&prms, 0, sizeof(parseParams));
+ prms.table = "rtevalruns";
+ prms.syskey = syskey;
+ prms.rterid = rterid;
+ prms.submid = submid;
+ prms.report_filename = report_fname;
+ rtevalrun_d = parseToSQLdata(dbc->log, xslt, summaryxml, &prms);
+ if( !rtevalrun_d ) {
+ writelog(dbc->log, LOG_ERR,
+ "[Connection %i] Could not parse the input XML data", dbc->id);
+ ret = -1;
+ goto exit;
+ }
+
+ // Register the rteval run information
+ dbdata = pgsql_INSERT(dbc, rtevalrun_d);
+ if( !dbdata ) {
+ ret = -1;
+ goto exit;
+ }
+
+ if( eCount(dbdata) != 1 ) {
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Failed to register the rteval run", dbc->id);
+ ret = -1;
+ eFree_values(dbdata);
+ goto exit;
+ }
+ eFree_values(dbdata);
+
+ // Parse the rtevalruns_details information
+ memset(&prms, 0, sizeof(parseParams));
+ prms.table = "rtevalruns_details";
+ prms.rterid = rterid;
+ rtevalrundets_d = parseToSQLdata(dbc->log, xslt, summaryxml, &prms);
+ if( !rtevalrundets_d ) {
+ writelog(dbc->log, LOG_ERR,
+ "[Connection %i] Could not parse the input XML data (rtevalruns_details)",
+ dbc->id);
+ ret = -1;
+ goto exit;
+ }
+
+ // Register the rteval_details information
+ dbdata = pgsql_INSERT(dbc, rtevalrundets_d);
+ if( !dbdata ) {
+ ret = -1;
+ goto exit;
+ }
+
+ // Check that only one record was inserted
+ if( eCount(dbdata) != 1 ) {
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] Failed to register the rteval run details", dbc->id);
+ ret = -1;
+ }
+ eFree_values(dbdata);
+ ret = 1;
+ exit:
+ if( rtevalrun_d ) {
+ xmlFreeDoc(rtevalrun_d);
+ }
+ if( rtevalrundets_d ) {
+ xmlFreeDoc(rtevalrundets_d);
+ }
+ return ret;
+}
+
+
+/**
+ * Registers data returned from cyclictest into the database.
+ *
+ * @param dbc Database handler where to perform the SQL queries
+ * @param xslt A pointer to a parsed 'xmlparser.xsl' XSLT template
+ * @param summaryxml The XML report from rteval
+ * @param rterid A positive integer referencing the rteval run ID, returned from db_register_rtevalrun()
+ *
+ * @return Returns 1 on success, otherwise -1
+ */
+int db_register_cyclictest(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml, int rterid) {
+ int result = -1;
+ xmlDoc *cyclic_d = NULL;
+ parseParams prms;
+ eurephiaVALUES *dbdata = NULL;
+ int cyclicdata = 0;
+ const char *cyclictables[] = { "cyclic_statistics", "cyclic_histogram", "cyclic_rawdata", NULL };
+ int i;
+
+ memset(&prms, 0, sizeof(parseParams));
+ prms.rterid = rterid;
+
+ // Register the cyclictest data
+ for( i = 0; cyclictables[i]; i++ ) {
+ prms.table = cyclictables[i];
+ cyclic_d = parseToSQLdata(dbc->log, xslt, summaryxml, &prms);
+ if( cyclic_d && cyclic_d->children ) {
+ // Insert SQL data which was found and generated
+ dbdata = pgsql_INSERT(dbc, cyclic_d);
+ if( !dbdata ) {
+ result = -1;
+ xmlFreeDoc(cyclic_d);
+ goto exit;
+ }
+
+ if (eCount(dbdata) > 0) {
+ cyclicdata++;
+ }
+ eFree_values(dbdata);
+ cyclicdata = 1;
+ }
+ if( cyclic_d ) {
+ xmlFreeDoc(cyclic_d);
+ }
+ }
+
+ // Report error if not enough cyclictest data is registered.
+ if( cyclicdata > 1 ) {
+ writelog(dbc->log, LOG_ALERT,
+ "[Connection %i] No cyclictest raw data or histogram data registered", dbc->id);
+ result = -1;
+ } else {
+ result = 1;
+ }
+ exit:
+ return result;
+}
diff --git a/server/parser/pgsql.h b/server/parser/pgsql.h
new file mode 100644
index 0000000..e40b7d2
--- /dev/null
+++ b/server/parser/pgsql.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file pgsql.h
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Wed Oct 13 17:44:35 2009
+ *
+ * @brief Database API for the PostgreSQL database.
+ *
+ *
+ */
+
+#ifndef _RTEVAL_PGSQL_H
+#define _RTEVAL_PGSQL_H
+
+#include <libpq-fe.h>
+#include <libxml/parser.h>
+#include <libxslt/transform.h>
+
+#include <log.h>
+#include <eurephia_values.h>
+#include <parsethread.h>
+
+/**
+ * A unified database abstraction layer, providing log support
+ */
+typedef struct {
+ unsigned int id; /**< Unique connection ID, used for debugging */
+ LogContext *log; /**< Initialised log context */
+ PGconn *db; /**< Database connection handler */
+} dbconn;
+
+/* Generic database function */
+dbconn *db_connect(eurephiaVALUES *cfg, unsigned int id, LogContext *log);
+int db_ping(dbconn *dbc);
+void db_disconnect(dbconn *dbc);
+int db_begin(dbconn *dbc);
+int db_commit(dbconn *dbc);
+int db_rollback(dbconn *dbc);
+
+/* rteval specific database functions */
+int db_wait_notification(dbconn *dbc, const int *shutdown, const char *listenfor);
+parseJob_t *db_get_submissionqueue_job(dbconn *dbc, pthread_mutex_t *mtx);
+int db_update_submissionqueue(dbconn *dbc, unsigned int submid, int status);
+int db_register_system(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml);
+int db_get_new_rterid(dbconn *dbc);
+int db_register_rtevalrun(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml,
+ unsigned int submid, int syskey, int rterid, const char *report_fname);
+int db_register_cyclictest(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml, int rterid);
+
+#endif
diff --git a/server/parser/rteval_parserd.c b/server/parser/rteval_parserd.c
new file mode 100644
index 0000000..a075ff0
--- /dev/null
+++ b/server/parser/rteval_parserd.c
@@ -0,0 +1,521 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file rteval_parserd.c
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Thu Oct 15 11:59:27 2009
+ *
+ * @brief Polls the rteval.submissionqueue table for notifications
+ * from new inserts and sends the file to a processing thread
+ *
+ *
+ *
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <signal.h>
+#include <pthread.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <assert.h>
+
+#include <eurephia_nullsafe.h>
+#include <eurephia_values.h>
+#include <configparser.h>
+#include <pgsql.h>
+#include <threadinfo.h>
+#include <parsethread.h>
+#include <argparser.h>
+
+#define DEFAULT_MSG_MAX 5 /**< Default size of the message queue */
+#define XMLPARSER_XSL "xmlparser.xsl" /**< rteval report parser XSLT, parses XML into database friendly data*/
+
+static int shutdown = 0; /**< Variable indicating if the program should shutdown */
+static LogContext *logctx = NULL; /**< Initialsed log context, to be used by sigcatch() */
+
+
+/**
+ * Simple signal catcher. Used for SIGINT and SIGTERM signals, and will set the global shutdown
+ * shutdown flag. It's expected that all threads behaves properly and exits as soon as their current
+ * work is completed
+ *
+ * @param sig Recieved signal (not used)
+ */
+void sigcatch(int sig) {
+ switch( sig ) {
+ case SIGINT:
+ case SIGTERM:
+ if( shutdown == 0 ) {
+ shutdown = 1;
+ writelog(logctx, LOG_INFO, "[SIGNAL] Shutting down");
+ } else {
+ writelog(logctx, LOG_INFO, "[SIGNAL] Shutdown in progress ... please be patient ...");
+ }
+ break;
+
+ case SIGUSR1:
+ writelog(logctx, LOG_EMERG, "[SIGNAL] Shutdown alarm from a worker thread");
+ shutdown = 1;
+ break;
+
+ default:
+ break;
+ }
+
+ // re-enable signals, to avoid brute force exits.
+ // If brute force is needed, SIGKILL is available.
+ signal(sig, sigcatch);
+}
+
+
+/**
+ * Opens and reads /proc/sys/fs/mqueue/msg_max, to get the maximum number of allowed messages
+ * on POSIX MQ queues. rteval_parserd will use as much of this as possible when needed.
+ *
+ * @return Returns the system msg_max value, or DEFAULT_MSG_MAX on failure to read the setting.
+ */
+unsigned int get_mqueue_msg_max(LogContext *log) {
+ FILE *fp = NULL;
+ char buf[130];
+ unsigned int msg_max = DEFAULT_MSG_MAX;
+
+ fp = fopen("/proc/sys/fs/mqueue/msg_max", "r");
+ if( !fp ) {
+ writelog(log, LOG_WARNING,
+ "Could not open /proc/sys/fs/mqueue/msg_max, defaulting to %i",
+ msg_max);
+ writelog(log, LOG_INFO, "%s", strerror(errno));
+ return msg_max;
+ }
+
+ memset(&buf, 0, 130);
+ if( fread(&buf, 1, 128, fp) < 1 ) {
+ writelog(log, LOG_WARNING,
+ "Could not read /proc/sys/fs/mqueue/msg_max, defaulting to %i",
+ msg_max);
+ writelog(log, LOG_INFO, "%s", strerror(errno));
+ } else {
+ msg_max = atoi_nullsafe(buf);
+ if( msg_max < 1 ) {
+ msg_max = DEFAULT_MSG_MAX;
+ writelog(log, LOG_WARNING,
+ "Failed to parse /proc/sys/fs/mqueue/msg_max,"
+ "defaulting to %i", msg_max);
+ }
+ }
+ fclose(fp);
+ return msg_max;
+}
+
+
+/**
+ * Main loop, which polls the submissionqueue table and puts jobs found here into a POSIX MQ queue
+ * which the worker threads will pick up.
+ *
+ * @param dbc Database connection, where to query the submission queue
+ * @param msgq file descriptor for the message queue
+ *
+ * @return Returns 0 on successful run, otherwise > 0 on errors.
+ */
+int process_submission_queue(dbconn *dbc, mqd_t msgq, int *activethreads) {
+ pthread_mutex_t mtx_submq = PTHREAD_MUTEX_INITIALIZER;
+ parseJob_t *job = NULL;
+ int rc = 0, i, actthr_cp = 0;
+
+ while( shutdown == 0 ) {
+ // Check status if the worker threads
+ // If we don't have any worker threads, shut down immediately
+ writelog(dbc->log, LOG_DEBUG, "Active worker threads: %i", *activethreads);
+ if( *activethreads < 1 ) {
+ writelog(dbc->log, LOG_EMERG,
+ "All worker threads ceased to exist. Shutting down!");
+ shutdown = 1;
+ rc = 1;
+ goto exit;
+ }
+
+ if( db_ping(dbc) != 1 ) {
+ writelog(dbc->log, LOG_EMERG, "Lost connection to database. Shutting down!");
+ shutdown = 1;
+ rc = 1;
+ goto exit;
+ }
+
+ // Fetch an available job
+ job = db_get_submissionqueue_job(dbc, &mtx_submq);
+ if( !job ) {
+ writelog(dbc->log, LOG_EMERG,
+ "Failed to get submission queue job. Shutting down!");
+ shutdown = 1;
+ rc = 1;
+ goto exit;
+ }
+ if( job->status == jbNONE ) {
+ free_nullsafe(job);
+ if( db_wait_notification(dbc, &shutdown, "rteval_submq") < 1 ) {
+ writelog(dbc->log, LOG_EMERG,
+ "Failed to wait for DB notification. Shutting down!");
+ shutdown = 1;
+ rc = 1;
+ goto exit;
+ }
+ continue;
+ }
+
+ // Send the job to the queue
+ writelog(dbc->log, LOG_INFO, "** New job: submid %i, %s", job->submid, job->filename);
+ do {
+ int res;
+
+ errno = 0;
+ res = mq_send(msgq, (char *) job, sizeof(parseJob_t), 1);
+ if( (res < 0) && (errno != EAGAIN) ) {
+ writelog(dbc->log, LOG_EMERG,
+ "Could not send parse job to the queue. "
+ "Shutting down!");
+ shutdown = 1;
+ rc = 2;
+ goto exit;
+ } else if( errno == EAGAIN ) {
+ writelog(dbc->log, LOG_WARNING,
+ "Message queue filled up. "
+ "Will not add new messages to queue for the next 60 seconds");
+ sleep(60);
+ }
+ } while( (errno == EAGAIN) );
+ free_nullsafe(job);
+ }
+
+ exit:
+ // Send empty messages to the threads, to make them have a look at the shutdown flag
+ job = (parseJob_t *) malloc_nullsafe(dbc->log, sizeof(parseJob_t));
+ errno = 0;
+ // Need to make a copy, as *activethreads will change when threads completes shutdown
+ actthr_cp = *activethreads;
+ for( i = 0; i < actthr_cp; i++ ) {
+ do {
+ int res;
+
+ writelog(dbc->log, LOG_DEBUG, "%s shutdown message %i of %i",
+ (errno == EAGAIN ? "Resending" : "Sending"), i+1, *activethreads);
+ errno = 0;
+ res = mq_send(msgq, (char *) job, sizeof(parseJob_t), 1);
+ if( (res < 0) && (errno != EAGAIN) ) {
+ writelog(dbc->log, LOG_EMERG,
+ "Could not send parse job to the queue. "
+ "Shutting down!");
+ shutdown = 1;
+ return rc;
+ } else if( errno == EAGAIN ) {
+ writelog(dbc->log, LOG_WARNING,
+ "Message queue filled up. "
+ "Will not add new messages to queue for the next 10 seconds");
+ sleep(10);
+ }
+ } while( (errno == EAGAIN) );
+ }
+ free_nullsafe(job);
+ return rc;
+}
+
+
+/**
+ * Prepares the program to be daemonised
+ *
+ * @param log Initialised log context, where log info of the process is reported
+ *
+ * @return Returns 1 on success, otherwise -1
+ */
+int daemonise(LogContext *log) {
+ pid_t pid, sid;
+ int i = 0;
+
+ if( (log->logtype == ltCONSOLE) ) {
+ writelog(log, LOG_EMERG,
+ "Cannot daemonise when logging to a console (stdout: or stderr:)");
+ return -1;
+ }
+
+ pid = fork();
+ if (pid < 0) {
+ writelog(log, LOG_EMERG, "Failed to daemonise the process (fork)");
+ return -1;
+ } else if (pid > 0) {
+ writelog(log, LOG_INFO, "Daemon pid: %ld", pid);
+ exit(EXIT_SUCCESS);
+ }
+
+ umask(0);
+
+ sid = setsid();
+ if (sid < 0) {
+ writelog(log, LOG_EMERG, "Failed to daemonise the process (setsid)");
+ return -1;
+ }
+
+ if ((chdir("/")) < 0) {
+ writelog(log, LOG_EMERG, "Failed to daemonise the process (fork)");
+ return -1;
+ }
+
+ // Prepare stdin, stdout and stderr for daemon mode
+ close(2);
+ close(1);
+ close(0);
+ i = open("/dev/null", O_RDWR); /* open stdin */
+ dup(i); /* stdout */
+ dup(i); /* stderr */
+
+ writelog(log, LOG_INFO, "Daemonised successfully");
+ return 1;
+}
+
+
+/**
+ * rtevald_parser main function.
+ *
+ * @param argc
+ * @param argv
+ *
+ * @return Returns the result of the process_submission_queue() function.
+ */
+int main(int argc, char **argv) {
+ eurephiaVALUES *config = NULL, *prgargs = NULL;
+ char xsltfile[2050], *reportdir = NULL;
+ xsltStylesheet *xslt = NULL;
+ dbconn *dbc = NULL;
+ pthread_t **threads = NULL;
+ pthread_attr_t **thread_attrs = NULL;
+ pthread_mutex_t mtx_sysreg = PTHREAD_MUTEX_INITIALIZER;
+ pthread_mutex_t mtx_thrcnt = PTHREAD_MUTEX_INITIALIZER;
+ threadData_t **thrdata = NULL;
+ struct mq_attr msgq_attr;
+ mqd_t msgq = 0;
+ int i,rc, mq_init = 0, max_threads = 0, started_threads = 0, activethreads = 0;
+
+ // Initialise XML and XSLT libraries
+ xsltInit();
+ xmlInitParser();
+
+ prgargs = parse_arguments(argc, argv);
+ if( prgargs == NULL ) {
+ fprintf(stderr, "** ERROR ** Failed to parse program arguments\n");
+ rc = 2;
+ goto exit;
+ }
+
+ // Setup a log context
+ logctx = init_log(eGet_value(prgargs, "log"), eGet_value(prgargs, "loglevel"));
+ if( !logctx ) {
+ fprintf(stderr, "** ERROR ** Could not setup a log context\n");
+ eFree_values(prgargs);
+ rc = 2;
+ goto exit;
+ }
+
+ // Fetch configuration
+ config = read_config(logctx, prgargs, "xmlrpc_parser");
+ eFree_values(prgargs); // read_config() copies prgargs into config, we don't need prgargs anymore
+
+ // Daemonise process if requested
+ if( atoi_nullsafe(eGet_value(config, "daemon")) == 1 ) {
+ if( daemonise(logctx) < 1 ) {
+ rc = 3;
+ goto exit;
+ }
+ }
+
+
+ // Parse XSLT template
+ snprintf(xsltfile, 512, "%s/%s", eGet_value(config, "xsltpath"), XMLPARSER_XSL);
+ writelog(logctx, LOG_DEBUG, "Parsing XSLT file: %s", xsltfile);
+ xslt = xsltParseStylesheetFile((xmlChar *) xsltfile);
+ if( !xslt ) {
+ writelog(logctx, LOG_EMERG, "Could not parse XSLT template: %s", xsltfile);
+ rc = 2;
+ goto exit;
+ }
+
+ // Open a POSIX MQ
+ writelog(logctx, LOG_DEBUG, "Preparing POSIX MQ queue: /rteval_parsequeue");
+ memset(&msgq, 0, sizeof(mqd_t));
+ msgq_attr.mq_maxmsg = get_mqueue_msg_max(logctx);
+ msgq_attr.mq_msgsize = sizeof(parseJob_t);
+ msgq_attr.mq_flags = O_NONBLOCK;
+ msgq = mq_open("/rteval_parsequeue", O_RDWR | O_CREAT, 0600, &msgq_attr);
+ if( msgq < 0 ) {
+ writelog(logctx, LOG_EMERG,
+ "Could not open message queue: %s", strerror(errno));
+ rc = 2;
+ goto exit;
+ }
+ mq_init = 1;
+
+ // Get the number of worker threads
+ max_threads = atoi_nullsafe(eGet_value(config, "threads"));
+ if( max_threads == 0 ) {
+ max_threads = 4;
+ }
+
+ // Get a database connection for the main thread
+ dbc = db_connect(config, max_threads, logctx);
+ if( !dbc ) {
+ rc = 4;
+ goto exit;
+ }
+
+ // Prepare all threads
+ threads = calloc(max_threads + 1, sizeof(pthread_t *));
+ thread_attrs = calloc(max_threads + 1, sizeof(pthread_attr_t *));
+ thrdata = calloc(max_threads + 1, sizeof(threadData_t *));
+ assert( (threads != NULL) && (thread_attrs != NULL) && (thrdata != NULL) );
+
+ reportdir = eGet_value(config, "reportdir");
+ writelog(logctx, LOG_INFO, "Starting %i worker threads", max_threads);
+ for( i = 0; i < max_threads; i++ ) {
+ // Prepare thread specific data
+ thrdata[i] = malloc_nullsafe(logctx, sizeof(threadData_t));
+ if( !thrdata[i] ) {
+ writelog(logctx, LOG_EMERG,
+ "Could not allocate memory for thread data");
+ rc = 2;
+ goto exit;
+ }
+
+ // Get a database connection for the thread
+ thrdata[i]->dbc = db_connect(config, i, logctx);
+ if( !thrdata[i]->dbc ) {
+ writelog(logctx, LOG_EMERG,
+ "Could not connect to the database for thread %i", i);
+ rc = 2;
+ shutdown = 1;
+ goto exit;
+ }
+
+ thrdata[i]->shutdown = &shutdown;
+ thrdata[i]->threadcount = &activethreads;
+ thrdata[i]->mtx_thrcnt = &mtx_thrcnt;
+ thrdata[i]->id = i;
+ thrdata[i]->msgq = msgq;
+ thrdata[i]->mtx_sysreg = &mtx_sysreg;
+ thrdata[i]->xslt = xslt;
+ thrdata[i]->destdir = reportdir;
+
+ thread_attrs[i] = malloc_nullsafe(logctx, sizeof(pthread_attr_t));
+ if( !thread_attrs[i] ) {
+ writelog(logctx, LOG_EMERG,
+ "Could not allocate memory for thread attributes");
+ rc = 2;
+ goto exit;
+ }
+ pthread_attr_init(thread_attrs[i]);
+ pthread_attr_setdetachstate(thread_attrs[i], PTHREAD_CREATE_JOINABLE);
+
+ threads[i] = malloc_nullsafe(logctx, sizeof(pthread_t));
+ if( !threads[i] ) {
+ writelog(logctx, LOG_EMERG,
+ "Could not allocate memory for pthread_t");
+ rc = 2;
+ goto exit;
+ }
+ }
+
+ // Setup signal catching
+ signal(SIGINT, sigcatch);
+ signal(SIGTERM, sigcatch);
+ signal(SIGHUP, SIG_IGN);
+ signal(SIGUSR1, sigcatch);
+ signal(SIGUSR2, SIG_IGN);
+
+ // Start the threads
+ for( i = 0; i < max_threads; i++ ) {
+ int thr_rc = pthread_create(threads[i], thread_attrs[i], parsethread, thrdata[i]);
+ if( thr_rc < 0 ) {
+ writelog(logctx, LOG_EMERG,
+ "** ERROR ** Failed to start thread %i: %s",
+ i, strerror(thr_rc));
+ rc = 3;
+ goto exit;
+ }
+ started_threads++;
+ }
+
+ // Main routine
+ //
+ // checks the submission queue and puts unprocessed records on the POSIX MQ
+ // to be parsed by one of the threads
+ //
+ writelog(logctx, LOG_DEBUG, "Starting submission queue checker");
+ rc = process_submission_queue(dbc, msgq, &activethreads);
+ writelog(logctx, LOG_DEBUG, "Submission queue checker shut down");
+
+ exit:
+ // Clean up all threads
+ for( i = 0; i < max_threads; i++ ) {
+ // Wait for all threads to exit
+ if( (i < started_threads) && threads && threads[i] ) {
+ void *thread_rc;
+ int j_rc;
+
+ if( (j_rc = pthread_join(*threads[i], &thread_rc)) != 0 ) {
+ writelog(logctx, LOG_CRIT,
+ "Failed to join thread %i: %s",
+ i, strerror(j_rc));
+ }
+ pthread_attr_destroy(thread_attrs[i]);
+ }
+ free_nullsafe(threads[i]);
+ free_nullsafe(thread_attrs[i]);
+
+ // Disconnect threads database connection
+ if( thrdata && thrdata[i] ) {
+ db_disconnect(thrdata[i]->dbc);
+ free_nullsafe(thrdata[i]);
+ }
+ }
+ free_nullsafe(thrdata);
+ free_nullsafe(threads);
+ free_nullsafe(thread_attrs);
+
+ // Close message queue
+ if( mq_init == 1 ) {
+ errno = 0;
+ if( mq_close(msgq) < 0 ) {
+ writelog(logctx, LOG_CRIT, "Failed to close message queue: %s",
+ strerror(errno));
+ }
+ errno = 0;
+ if( mq_unlink("/rteval_parsequeue") < 0 ) {
+ writelog(logctx, LOG_ALERT, "Failed to remove the message queue: %s",
+ strerror(errno));
+ }
+ }
+
+ // Disconnect from database, main thread connection
+ db_disconnect(dbc);
+
+ // Free up the rest
+ eFree_values(config);
+ xsltFreeStylesheet(xslt);
+ xmlCleanupParser();
+ xsltCleanupGlobals();
+
+ writelog(logctx, LOG_EMERG, "rteval_parserd is stopped");
+ close_log(logctx);
+ return rc;
+}
+
diff --git a/server/parser/sha1.c b/server/parser/sha1.c
new file mode 100644
index 0000000..3f77aa9
--- /dev/null
+++ b/server/parser/sha1.c
@@ -0,0 +1,615 @@
+/*-
+ * Copyright (c) 2001-2003 Allan Saddi <allan@saddi.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY ALLAN SADDI AND HIS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL ALLAN SADDI OR HIS CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $Id: sha1.c 680 2003-07-25 21:57:38Z asaddi $
+ */
+
+/*
+ * Define WORDS_BIGENDIAN if compiling on a big-endian architecture.
+ *
+ * Define SHA1_TEST to test the implementation using the NIST's
+ * sample messages. The output should be:
+ *
+ * a9993e36 4706816a ba3e2571 7850c26c 9cd0d89d
+ * 84983e44 1c3bd26e baae4aa1 f95129e5 e54670f1
+ * 34aa973c d4c4daa4 f61eeb2b dbad2731 6534016f
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+#include "sha1.h"
+
+#ifndef lint
+static const char rcsid[] =
+ "$Id: sha1.c 680 2003-07-25 21:57:38Z asaddi $";
+#endif /* !lint */
+
+#define ROTL(x, n) (((x) << (n)) | ((x) >> (32 - (n))))
+#define ROTR(x, n) (((x) >> (n)) | ((x) << (32 - (n))))
+
+#define F_0_19(x, y, z) ((z) ^ ((x) & ((y) ^ (z))))
+#define F_20_39(x, y, z) ((x) ^ (y) ^ (z))
+#define F_40_59(x, y, z) (((x) & ((y) | (z))) | ((y) & (z)))
+#define F_60_79(x, y, z) ((x) ^ (y) ^ (z))
+
+#define DO_ROUND(F, K) { \
+ temp = ROTL(a, 5) + F(b, c, d) + e + *(W++) + K; \
+ e = d; \
+ d = c; \
+ c = ROTL(b, 30); \
+ b = a; \
+ a = temp; \
+}
+
+#define K_0_19 0x5a827999L
+#define K_20_39 0x6ed9eba1L
+#define K_40_59 0x8f1bbcdcL
+#define K_60_79 0xca62c1d6L
+
+#ifndef RUNTIME_ENDIAN
+
+#ifdef WORDS_BIGENDIAN
+
+#define BYTESWAP(x) (x)
+#define BYTESWAP64(x) (x)
+
+#else /* WORDS_BIGENDIAN */
+
+#define BYTESWAP(x) ((ROTR((x), 8) & 0xff00ff00L) | \
+ (ROTL((x), 8) & 0x00ff00ffL))
+#define BYTESWAP64(x) _byteswap64(x)
+
+static inline uint64_t _byteswap64(uint64_t x)
+{
+ uint32_t a = x >> 32;
+ uint32_t b = (uint32_t) x;
+ return ((uint64_t) BYTESWAP(b) << 32) | (uint64_t) BYTESWAP(a);
+}
+
+#endif /* WORDS_BIGENDIAN */
+
+#else /* !RUNTIME_ENDIAN */
+
+#define BYTESWAP(x) _byteswap(sc->littleEndian, x)
+#define BYTESWAP64(x) _byteswap64(sc->littleEndian, x)
+
+#define _BYTESWAP(x) ((ROTR((x), 8) & 0xff00ff00L) | \
+ (ROTL((x), 8) & 0x00ff00ffL))
+#define _BYTESWAP64(x) __byteswap64(x)
+
+static inline uint64_t __byteswap64(uint64_t x)
+{
+ uint32_t a = x >> 32;
+ uint32_t b = (uint32_t) x;
+ return ((uint64_t) _BYTESWAP(b) << 32) | (uint64_t) _BYTESWAP(a);
+}
+
+static inline uint32_t _byteswap(int littleEndian, uint32_t x)
+{
+ if (!littleEndian)
+ return x;
+ else
+ return _BYTESWAP(x);
+}
+
+static inline uint64_t _byteswap64(int littleEndian, uint64_t x)
+{
+ if (!littleEndian)
+ return x;
+ else
+ return _BYTESWAP64(x);
+}
+
+static inline void setEndian(int *littleEndianp)
+{
+ union {
+ uint32_t w;
+ uint8_t b[4];
+ } endian;
+
+ endian.w = 1L;
+ *littleEndianp = endian.b[0] != 0;
+}
+
+#endif /* !RUNTIME_ENDIAN */
+
+static const uint8_t padding[64] = {
+ 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+void
+SHA1Init (SHA1Context *sc)
+{
+#ifdef RUNTIME_ENDIAN
+ setEndian (&sc->littleEndian);
+#endif /* RUNTIME_ENDIAN */
+
+ sc->totalLength = 0LL;
+ sc->hash[0] = 0x67452301L;
+ sc->hash[1] = 0xefcdab89L;
+ sc->hash[2] = 0x98badcfeL;
+ sc->hash[3] = 0x10325476L;
+ sc->hash[4] = 0xc3d2e1f0L;
+ sc->bufferLength = 0L;
+}
+
+static void
+burnStack (int size)
+{
+ char buf[128];
+
+ memset (buf, 0, sizeof (buf));
+ size -= sizeof (buf);
+ if (size > 0)
+ burnStack (size);
+}
+
+static void
+SHA1Guts (SHA1Context *sc, const uint32_t *cbuf)
+{
+ uint32_t buf[80];
+ uint32_t *W, *W3, *W8, *W14, *W16;
+ uint32_t a, b, c, d, e, temp;
+ int i;
+
+ W = buf;
+
+ for (i = 15; i >= 0; i--) {
+ *(W++) = BYTESWAP(*cbuf);
+ cbuf++;
+ }
+
+ W16 = &buf[0];
+ W14 = &buf[2];
+ W8 = &buf[8];
+ W3 = &buf[13];
+
+ for (i = 63; i >= 0; i--) {
+ *W = *(W3++) ^ *(W8++) ^ *(W14++) ^ *(W16++);
+ *W = ROTL(*W, 1);
+ W++;
+ }
+
+ a = sc->hash[0];
+ b = sc->hash[1];
+ c = sc->hash[2];
+ d = sc->hash[3];
+ e = sc->hash[4];
+
+ W = buf;
+
+#ifndef SHA1_UNROLL
+#define SHA1_UNROLL 20
+#endif /* !SHA1_UNROLL */
+
+#if SHA1_UNROLL == 1
+ for (i = 19; i >= 0; i--)
+ DO_ROUND(F_0_19, K_0_19);
+
+ for (i = 19; i >= 0; i--)
+ DO_ROUND(F_20_39, K_20_39);
+
+ for (i = 19; i >= 0; i--)
+ DO_ROUND(F_40_59, K_40_59);
+
+ for (i = 19; i >= 0; i--)
+ DO_ROUND(F_60_79, K_60_79);
+#elif SHA1_UNROLL == 2
+ for (i = 9; i >= 0; i--) {
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ }
+
+ for (i = 9; i >= 0; i--) {
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ }
+
+ for (i = 9; i >= 0; i--) {
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ }
+
+ for (i = 9; i >= 0; i--) {
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ }
+#elif SHA1_UNROLL == 4
+ for (i = 4; i >= 0; i--) {
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ }
+
+ for (i = 4; i >= 0; i--) {
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ }
+
+ for (i = 4; i >= 0; i--) {
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ }
+
+ for (i = 4; i >= 0; i--) {
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ }
+#elif SHA1_UNROLL == 5
+ for (i = 3; i >= 0; i--) {
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ }
+
+ for (i = 3; i >= 0; i--) {
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ }
+
+ for (i = 3; i >= 0; i--) {
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ }
+
+ for (i = 3; i >= 0; i--) {
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ }
+#elif SHA1_UNROLL == 10
+ for (i = 1; i >= 0; i--) {
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ }
+
+ for (i = 1; i >= 0; i--) {
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ }
+
+ for (i = 1; i >= 0; i--) {
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ }
+
+ for (i = 1; i >= 0; i--) {
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ }
+#elif SHA1_UNROLL == 20
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+ DO_ROUND(F_0_19, K_0_19);
+
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+ DO_ROUND(F_20_39, K_20_39);
+
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+ DO_ROUND(F_40_59, K_40_59);
+
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+ DO_ROUND(F_60_79, K_60_79);
+#else /* SHA1_UNROLL */
+#error SHA1_UNROLL must be 1, 2, 4, 5, 10 or 20!
+#endif
+
+ sc->hash[0] += a;
+ sc->hash[1] += b;
+ sc->hash[2] += c;
+ sc->hash[3] += d;
+ sc->hash[4] += e;
+}
+
+void
+SHA1Update (SHA1Context *sc, const void *vdata, uint32_t len)
+{
+ const uint8_t *data = vdata;
+ uint32_t bufferBytesLeft;
+ uint32_t bytesToCopy;
+ int needBurn = 0;
+
+#ifdef SHA1_FAST_COPY
+ if (sc->bufferLength) {
+ bufferBytesLeft = 64L - sc->bufferLength;
+
+ bytesToCopy = bufferBytesLeft;
+ if (bytesToCopy > len)
+ bytesToCopy = len;
+
+ memcpy (&sc->buffer.bytes[sc->bufferLength], data, bytesToCopy);
+
+ sc->totalLength += bytesToCopy * 8L;
+
+ sc->bufferLength += bytesToCopy;
+ data += bytesToCopy;
+ len -= bytesToCopy;
+
+ if (sc->bufferLength == 64L) {
+ SHA1Guts (sc, sc->buffer.words);
+ needBurn = 1;
+ sc->bufferLength = 0L;
+ }
+ }
+
+ while (len > 63) {
+ sc->totalLength += 512L;
+
+ SHA1Guts (sc, data);
+ needBurn = 1;
+
+ data += 64L;
+ len -= 64L;
+ }
+
+ if (len) {
+ memcpy (&sc->buffer.bytes[sc->bufferLength], data, len);
+
+ sc->totalLength += len * 8L;
+
+ sc->bufferLength += len;
+ }
+#else /* SHA1_FAST_COPY */
+ while (len) {
+ bufferBytesLeft = 64L - sc->bufferLength;
+
+ bytesToCopy = bufferBytesLeft;
+ if (bytesToCopy > len)
+ bytesToCopy = len;
+
+ memcpy (&sc->buffer.bytes[sc->bufferLength], data, bytesToCopy);
+
+ sc->totalLength += bytesToCopy * 8L;
+
+ sc->bufferLength += bytesToCopy;
+ data += bytesToCopy;
+ len -= bytesToCopy;
+
+ if (sc->bufferLength == 64L) {
+ SHA1Guts (sc, sc->buffer.words);
+ needBurn = 1;
+ sc->bufferLength = 0L;
+ }
+ }
+#endif /* SHA1_FAST_COPY */
+
+ if (needBurn)
+ burnStack (sizeof (uint32_t[86]) + sizeof (uint32_t *[5]) + sizeof (int));
+}
+
+void
+SHA1Final (SHA1Context *sc, uint8_t hash[SHA1_HASH_SIZE])
+{
+ uint32_t bytesToPad;
+ uint64_t lengthPad;
+ int i;
+
+ bytesToPad = 120L - sc->bufferLength;
+ if (bytesToPad > 64L)
+ bytesToPad -= 64L;
+
+ lengthPad = BYTESWAP64(sc->totalLength);
+
+ SHA1Update (sc, padding, bytesToPad);
+ SHA1Update (sc, &lengthPad, 8L);
+
+ if (hash) {
+ for (i = 0; i < SHA1_HASH_WORDS; i++) {
+#ifdef SHA1_FAST_COPY
+ *((uint32_t *) hash) = BYTESWAP(sc->hash[i]);
+#else /* SHA1_FAST_COPY */
+ hash[0] = (uint8_t) (sc->hash[i] >> 24);
+ hash[1] = (uint8_t) (sc->hash[i] >> 16);
+ hash[2] = (uint8_t) (sc->hash[i] >> 8);
+ hash[3] = (uint8_t) sc->hash[i];
+#endif /* SHA1_FAST_COPY */
+ hash += 4;
+ }
+ }
+}
+
+#ifdef SHA1_TEST
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int
+main (int argc, char *argv[])
+{
+ SHA1Context foo;
+ uint8_t hash[SHA1_HASH_SIZE];
+ char buf[1000];
+ int i;
+
+ SHA1Init (&foo);
+ SHA1Update (&foo, "abc", 3);
+ SHA1Final (&foo, hash);
+
+ for (i = 0; i < SHA1_HASH_SIZE;) {
+ printf ("%02x", hash[i++]);
+ if (!(i % 4))
+ printf (" ");
+ }
+ printf ("\n");
+
+ SHA1Init (&foo);
+ SHA1Update (&foo,
+ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
+ 56);
+ SHA1Final (&foo, hash);
+
+ for (i = 0; i < SHA1_HASH_SIZE;) {
+ printf ("%02x", hash[i++]);
+ if (!(i % 4))
+ printf (" ");
+ }
+ printf ("\n");
+
+ SHA1Init (&foo);
+ memset (buf, 'a', sizeof (buf));
+ for (i = 0; i < 1000; i++)
+ SHA1Update (&foo, buf, sizeof (buf));
+ SHA1Final (&foo, hash);
+
+ for (i = 0; i < SHA1_HASH_SIZE;) {
+ printf ("%02x", hash[i++]);
+ if (!(i % 4))
+ printf (" ");
+ }
+ printf ("\n");
+
+ exit (0);
+}
+
+#endif /* SHA1_TEST */
diff --git a/server/parser/sha1.h b/server/parser/sha1.h
new file mode 100644
index 0000000..9ce5bd6
--- /dev/null
+++ b/server/parser/sha1.h
@@ -0,0 +1,66 @@
+/*-
+ * Copyright (c) 2001-2003 Allan Saddi <allan@saddi.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY ALLAN SADDI AND HIS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL ALLAN SADDI OR HIS CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $Id: sha1.h 347 2003-02-23 22:11:49Z asaddi $
+ */
+
+#ifndef _SHA1_H
+#define _SHA1_H
+
+#include <stdint.h>
+
+#define SHA1_HASH_SIZE 20
+
+/* Hash size in 32-bit words */
+#define SHA1_HASH_WORDS 5
+
+struct _SHA1Context {
+ uint64_t totalLength;
+ uint32_t hash[SHA1_HASH_WORDS];
+ uint32_t bufferLength;
+ union {
+ uint32_t words[16];
+ uint8_t bytes[64];
+ } buffer;
+#ifdef RUNTIME_ENDIAN
+ int littleEndian;
+#endif /* RUNTIME_ENDIAN */
+};
+
+typedef struct _SHA1Context SHA1Context;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void SHA1Init (SHA1Context *sc);
+void SHA1Update (SHA1Context *sc, const void *data, uint32_t len);
+void SHA1Final (SHA1Context *sc, uint8_t hash[SHA1_HASH_SIZE]);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SHA1_H */
diff --git a/server/parser/statuses.h b/server/parser/statuses.h
new file mode 100644
index 0000000..701c8e8
--- /dev/null
+++ b/server/parser/statuses.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file statuses.h
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Wed Oct 21 11:17:24 2009
+ *
+ * @brief Status values used by rteval_parserd
+ *
+ */
+
+#ifndef _RTEVAL_STATUS_H
+#define _RTEVAL_STATUS_H
+
+#define STAT_NEW 0 /**< New, unparsed report in the submission queue */
+#define STAT_ASSIGNED 1 /**< Submission is assigned to a parser */
+#define STAT_INPROG 2 /**< Parsing has started */
+#define STAT_SUCCESS 3 /**< Report parsed successfully */
+#define STAT_UNKNFAIL 4 /**< Unkown failure */
+#define STAT_XMLFAIL 5 /**< Failed to parse the report XML file */
+#define STAT_SYSREG 6 /**< System registration failed */
+#define STAT_RTERIDREG 7 /**< Failed to get a new rterid value for the rteval run */
+#define STAT_GENDB 8 /**< General database error */
+#define STAT_RTEVRUNS 9 /**< Registering rteval run information failed */
+#define STAT_CYCLIC 10 /**< Registering cyclictest results failed */
+#define STAT_REPMOVE 11 /**< Failed to move the report file */
+
+#endif
diff --git a/server/parser/threadinfo.h b/server/parser/threadinfo.h
new file mode 100644
index 0000000..d2bcf43
--- /dev/null
+++ b/server/parser/threadinfo.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file threadinfo.h
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Thu Oct 15 11:47:51 2009
+ *
+ * @brief Shared info between the main() and parsethread() functions
+ *
+ */
+
+#ifndef _THREADINFO_H
+#define _THREADINFO_H
+
+#include <mqueue.h>
+#include <libxslt/transform.h>
+
+/**
+ * Thread slot information. Each thread slot is assigned with one threadData_t element.
+ */
+typedef struct {
+ int *shutdown; /**< If set to 1, the thread should shut down */
+ int *threadcount; /**< Number of active worker threads */
+ pthread_mutex_t *mtx_thrcnt; /**< Mutex lock for updating active worker threads */
+ mqd_t msgq; /**< POSIX MQ descriptor */
+ pthread_mutex_t *mtx_sysreg; /**< Mutex locking, to avoid clashes with registering systems */
+ unsigned int id; /**< Numeric ID for this thread */
+ dbconn *dbc; /**< Database connection assigned to this thread */
+ xsltStylesheet *xslt; /**< XSLT stylesheet assigned to this thread */
+ const char *destdir; /**< Directory where to put the parsed reports */
+} threadData_t;
+
+#endif
diff --git a/server/parser/xmlparser.c b/server/parser/xmlparser.c
new file mode 100644
index 0000000..8cf13a8
--- /dev/null
+++ b/server/parser/xmlparser.c
@@ -0,0 +1,414 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * David Sommerseth <davids@redhat.com>
+ *
+ * Parses summary.xml reports from rteval into a standardised XML format
+ * which is useful when putting data into a database.
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file xmlparser.c
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Wed Oct 21 10:58:53 2009
+ *
+ * @brief Parses summary.xml reports from rteval into a standardised XML format
+ * which is useful when putting data into a database.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include <libxml/tree.h>
+#include <libxslt/xsltInternals.h>
+#include <libxslt/transform.h>
+#include <libxslt/xsltutils.h>
+
+#include <eurephia_nullsafe.h>
+#include <eurephia_xml.h>
+#include <xmlparser.h>
+#include <sha1.h>
+#include <log.h>
+
+/**
+ * Simple strdup() function which encapsulates the string in single quotes,
+ * which is needed for XSLT parameter values
+ *
+ * @param str The string to be strdup'ed and encapsulated
+ *
+ * @return Returns a pointer to the new buffer.
+ */
+static char *encapsString(const char *str) {
+ char *ret = NULL;
+
+ if( str == NULL ) {
+ return NULL;
+ }
+
+ ret = (char *) calloc(1, strlen(str)+4);
+ assert( ret != NULL );
+
+ snprintf(ret, strlen(str)+3, "'%s'", str);
+ return ret;
+}
+
+
+/**
+ * Converts an integer to string an encapsulates the value in single quotes,
+ * which is needed for XSLT parameter values.
+ *
+ * @param val Integer value to encapsulate
+ *
+ * @return Returns a pointer to a new buffer with the encapsulated integer value. This
+ * buffer must be free'd after usage.
+ */
+static char *encapsInt(const unsigned int val) {
+ char *buf = NULL;
+
+ buf = (char *) calloc(1, 130);
+ snprintf(buf, 128, "'%i'", val);
+ return buf;
+}
+
+
+/**
+ * Parses any XML input document into a sqldata XML format which can be used by pgsql_INSERT().
+ * The transformation must be defined in the input XSLT template.
+ *
+ * @param log Log context
+ * @param xslt XSLT template defining the data transformation
+ * @param indata_d Input XML data to transform to a sqldata XML document
+ * @param params Parameters to be sent to the XSLT parser
+ *
+ * @return Returns a well formed sqldata XML document on success, otherwise NULL is returned.
+ */
+xmlDoc *parseToSQLdata(LogContext *log, xsltStylesheet *xslt, xmlDoc *indata_d, parseParams *params) {
+ xmlDoc *result_d = NULL;
+ char *xsltparams[10];
+ unsigned int idx = 0, idx_table = 0, idx_submid = 0,
+ idx_syskey = 0, idx_rterid = 0, idx_repfname = 0;
+
+ if( params->table == NULL ) {
+ writelog(log, LOG_ERR, "Table is not defined");
+ return NULL;
+ }
+
+ // Prepare XSLT parameters
+ xsltparams[idx++] = "table\0";
+ xsltparams[idx] = (char *) encapsString(params->table);
+ idx_table = idx++;
+
+ if( params->submid > 0) {
+ xsltparams[idx++] = "submid\0";
+ xsltparams[idx] = (char *) encapsInt(params->submid);
+ idx_submid = idx++;
+ }
+
+ if( params->syskey > 0) {
+ xsltparams[idx++] = "syskey\0";
+ xsltparams[idx] = (char *) encapsInt(params->syskey);
+ idx_syskey = idx++;
+ }
+
+ if( params->rterid > 0 ) {
+ xsltparams[idx++] = "rterid";
+ xsltparams[idx] = (char *) encapsInt(params->rterid);
+ idx_rterid = idx++;
+ }
+
+ if( params->report_filename ) {
+ xsltparams[idx++] = "report_filename";
+ xsltparams[idx] = (char *) encapsString(params->report_filename);
+ idx_repfname = idx++;
+ }
+ xsltparams[idx] = NULL;
+
+ // Apply the XSLT template to the input XML data
+ result_d = xsltApplyStylesheet(xslt, indata_d, (const char **)xsltparams);
+ if( result_d == NULL ) {
+ writelog(log, LOG_CRIT, "Failed applying XSLT template to input XML");
+ }
+
+ // Free memory we allocated via encapsString()/encapsInt()
+ free(xsltparams[idx_table]);
+ if( params->submid ) {
+ free(xsltparams[idx_submid]);
+ }
+ if( params->syskey ) {
+ free(xsltparams[idx_syskey]);
+ }
+ if( params->rterid ) {
+ free(xsltparams[idx_rterid]);
+ }
+ if( params->report_filename ) {
+ free(xsltparams[idx_repfname]);
+ }
+
+ return result_d;
+}
+
+
+/**
+ * Internal xmlparser function. Extracts the value from a '//sqldata/records/record/value'
+ * node and hashes the value if the 'hash' attribute is set. Otherwise the value is extracted
+ * from the node directly. This function is only used by sqldataExtractContent().
+ *
+ * @param sql_n sqldata values node containing the value to extract.
+ *
+ * @return Returns a pointer to a new buffer containing the value on success, otherwise NULL.
+ * This memory buffer must be free'd after usage.
+ */
+static inline char *sqldataValueHash(LogContext *log, xmlNode *sql_n) {
+ const char *hash = NULL;
+ SHA1Context shactx;
+ uint8_t shahash[SHA1_HASH_SIZE];
+ char *ret = NULL, *ptr = NULL;;
+ int i;
+
+ if( !sql_n || (xmlStrcmp(sql_n->name, (xmlChar *) "value") != 0)
+ || (xmlStrcmp(sql_n->parent->name, (xmlChar *) "record") != 0) ) {
+ return NULL;
+ }
+
+ hash = xmlGetAttrValue(sql_n->properties, "hash");
+ if( !hash ) {
+ // If no hash attribute is found, just use the raw data
+ ret = strdup_nullsafe(xmlExtractContent(sql_n));
+ } else if( strcasecmp(hash, "sha1") == 0 ) {
+ const char *indata = xmlExtractContent(sql_n);
+ // SHA1 hashing requested
+ SHA1Init(&shactx);
+ SHA1Update(&shactx, indata, strlen_nullsafe(indata));
+ SHA1Final(&shactx, shahash);
+
+ // "Convert" to a readable format
+ ret = malloc_nullsafe(log, (SHA1_HASH_SIZE * 2) + 3);
+ ptr = ret;
+ for( i = 0; i < SHA1_HASH_SIZE; i++ ) {
+ sprintf(ptr, "%02x", shahash[i]);
+ ptr += 2;
+ }
+ } else {
+ ret = strdup("<Unsupported hashing algorithm>");
+ }
+
+ return ret;
+}
+
+
+/**
+ * Extract the content of a '//sqldata/records/record/value' node. It will consider
+ * both the 'hash' and 'type' attributes of the 'value' tag.
+ *
+ * @param log Log context
+ * @param sql_n Pointer to a value node of a sqldata XML document.
+ *
+ * @return Returns a pointer to a new memory buffer containing the value as a string.
+ * On errors, NULL is returned. This memory buffer must be free'd after usage.
+ */
+char *sqldataExtractContent(LogContext *log, xmlNode *sql_n) {
+ const char *valtype = xmlGetAttrValue(sql_n->properties, "type");
+
+ if( !sql_n || (xmlStrcmp(sql_n->name, (xmlChar *) "value") != 0)
+ || (xmlStrcmp(sql_n->parent->name, (xmlChar *) "record") != 0) ) {
+ return NULL;
+ }
+
+ if( valtype && (strcmp(valtype, "xmlblob") == 0) ) {
+ xmlNode *chld_n = sql_n->children;
+
+ // Go to next "real" tag, skipping non-element nodes
+ while( chld_n && chld_n->type != XML_ELEMENT_NODE ){
+ chld_n = chld_n->next;
+ }
+ return xmlNodeToString(log, chld_n);
+ } else {
+ return sqldataValueHash(log, sql_n);
+ }
+}
+
+
+/**
+ * Return the 'fid' value of a given field in an sqldata XML document.
+ *
+ * @param log Log context
+ * @param sql_n Pointer to the root xmlNode element of a sqldata XML document
+ * @param fname String containing the field name to look up
+ *
+ * @return Returns a value >= 0 on success, containing the 'fid' value of the field. Otherwise
+ * a value < 0 is returned. -1 if the field is not found or -2 if there are some problems
+ * with the XML document.
+ */
+int sqldataGetFid(LogContext *log, xmlNode *sql_n, const char *fname) {
+ xmlNode *f_n = NULL;
+
+ if( !sql_n || (xmlStrcmp(sql_n->name, (xmlChar *) "sqldata") != 0) ) {
+ writelog(log, LOG_ERR,
+ "sqldataGetFid: Input XML document is not a valid sqldata document");
+ return -2;
+ }
+
+ f_n = xmlFindNode(sql_n, "fields");
+ if( !f_n || !f_n->children ) {
+ writelog(log, LOG_ERR,
+ "sqldataGetFid: Input XML document does not contain a fields section");
+ return -2;
+ }
+
+ foreach_xmlnode(f_n->children, f_n) {
+ if( (f_n->type != XML_ELEMENT_NODE)
+ || xmlStrcmp(f_n->name, (xmlChar *) "field") != 0 ) {
+ // Skip uninteresting nodes
+ continue;
+ }
+
+ if( strcmp(xmlExtractContent(f_n), fname) == 0 ) {
+ char *fid = xmlGetAttrValue(f_n->properties, "fid");
+ if( !fid ) {
+ writelog(log, LOG_ERR,
+ "sqldataGetFid: Field node is missing 'fid' attribute (field: %s)",
+ fname);
+ return -2;
+ }
+ return atoi_nullsafe(fid);
+ }
+ }
+ return -1;
+}
+
+
+/**
+ * Retrieves the value of a particular field in an sqldata XML document.
+ *
+ * @param log Log context
+ * @param sqld pointer to an sqldata XML document.
+ * @param fname String containing the field name to extract the value of.
+ * @param recid Integer containing the record ID of the record to extract the value. This starts
+ * on 0.
+ *
+ * @return Returns a pointer to a new memory buffer containing the extracted value. On errors or if
+ * recid is higher than available records, NULL is returned.
+ */
+char *sqldataGetValue(LogContext *log, xmlDoc *sqld, const char *fname, int recid ) {
+ xmlNode *r_n = NULL;
+ int fid = -3, rc = 0;
+
+ if( recid < 0 ) {
+ writelog(log, LOG_ERR, "sqldataGetValue: Invalid recid");
+ return NULL;
+ }
+
+ r_n = xmlDocGetRootElement(sqld);
+ if( !r_n || (xmlStrcmp(r_n->name, (xmlChar *) "sqldata") != 0) ) {
+ writelog(log, LOG_ERR,
+ "sqldataGetValue: Input XML document is not a valid sqldata document");
+ return NULL;
+ }
+
+ fid = sqldataGetFid(log, r_n, fname);
+ if( fid < 0 ) {
+ return NULL;
+ }
+
+ r_n = xmlFindNode(r_n, "records");
+ if( !r_n || !r_n->children ) {
+ writelog(log, LOG_ERR,
+ "sqldataGetValue: Input XML document does not contain a records section");
+ return NULL;
+ }
+
+ foreach_xmlnode(r_n->children, r_n) {
+ if( (r_n->type != XML_ELEMENT_NODE)
+ || xmlStrcmp(r_n->name, (xmlChar *) "record") != 0 ) {
+ // Skip uninteresting nodes
+ continue;
+ }
+ if( rc == recid ) {
+ xmlNode *v_n = NULL;
+ // The rigth record is found, find the field we're looking for
+ foreach_xmlnode(r_n->children, v_n) {
+ char *fid_s = NULL;
+ if( (v_n->type != XML_ELEMENT_NODE)
+ || (xmlStrcmp(v_n->name, (xmlChar *) "value") != 0) ) {
+ // Skip uninteresting nodes
+ continue;
+ }
+ fid_s = xmlGetAttrValue(v_n->properties, "fid");
+ if( fid_s && (fid == atoi_nullsafe(fid_s)) ) {
+ return sqldataExtractContent(log, v_n);
+ }
+ }
+ }
+ rc++;
+ }
+ return NULL;
+}
+
+
+/**
+ * Helper function to parse an sqldata XML document for the systems_hostname table. In addition
+ * it will also return two strings containing hostname and ipaddress of the host.
+ *
+ * @param log Log context
+ * @param xslt Pointer to an xmlparser.xml XSLT template
+ * @param summaryxml rteval XML report document
+ * @param syskey Integer containing the syskey value corresponding to this host
+ * @param hostname Return pointer for where the hostname will be saved.
+ * @param ipaddr Return pointer for where the IP address will be saved.
+ *
+ * @return Returns a sqldata XML document on success. In this case the hostname and ipaddr will point
+ * at memory buffers containing hostname and ipaddress. These values must be free'd after usage.
+ * On errors the function will return NULL and hostname and ipaddr will not have been touched
+ * at all.
+ */
+xmlDoc *sqldataGetHostInfo(LogContext *log, xsltStylesheet *xslt, xmlDoc *summaryxml,
+ int syskey, char **hostname, char **ipaddr)
+{
+ xmlDoc *hostinfo_d = NULL;
+ parseParams prms;
+
+ memset(&prms, 0, sizeof(parseParams));
+ prms.table = "systems_hostname";
+ prms.syskey = syskey;
+
+ hostinfo_d = parseToSQLdata(log, xslt, summaryxml, &prms);
+ if( !hostinfo_d ) {
+ writelog(log, LOG_ERR,
+ "sqldatGetHostInfo: Could not parse input XML data");
+ xmlFreeDoc(hostinfo_d);
+ goto exit;
+ }
+
+ // Grab hostname from input XML
+ *hostname = sqldataGetValue(log, hostinfo_d, "hostname", 0);
+ if( !hostname ) {
+ writelog(log, LOG_ERR,
+ "sqldatGetHostInfo: Could not retrieve the hostname field from the input XML");
+ xmlFreeDoc(hostinfo_d);
+ goto exit;
+ }
+
+ // Grab ipaddr from input XML
+ *ipaddr = sqldataGetValue(log, hostinfo_d, "ipaddr", 0);
+ if( !ipaddr ) {
+ writelog(log, LOG_ERR,
+ "sqldatGetHostInfo: Could not retrieve the IP address field from the input XML");
+ free_nullsafe(hostname);
+ xmlFreeDoc(hostinfo_d);
+ goto exit;
+ }
+ exit:
+ return hostinfo_d;
+}
diff --git a/server/parser/xmlparser.h b/server/parser/xmlparser.h
new file mode 100644
index 0000000..2c96fdf
--- /dev/null
+++ b/server/parser/xmlparser.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2009 Red Hat Inc.
+ *
+ * David Sommerseth <davids@redhat.com>
+ *
+ *
+ * This application is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2.
+ *
+ * This application is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+/**
+ * @file xmlparser.h
+ * @author David Sommerseth <davids@redhat.com>
+ * @date Wed Oct 7 17:27:39 2009
+ *
+ * @brief Parses summary.xml reports from rteval into a standardised XML format
+ * which is useful when putting data into a database.
+ *
+ */
+
+
+#ifndef _XMLPARSER_H
+#define _XMLPARSER_H
+
+/**
+ * Parameters needed by the the xmlparser.xsl XSLT template.
+ */
+typedef struct {
+ const char *table; /**< Which table to parse data for. Required*/
+ unsigned int submid; /**< Submission ID, needed by the 'rtevalruns' table */
+ unsigned int syskey; /**< System key (referencing systems.syskey) */
+ const char *report_filename; /**< Filename to the saved report (after being parsed) */
+ unsigned int rterid; /**< References rtevalruns.rterid */
+} parseParams;
+
+xmlDoc *parseToSQLdata(LogContext *log, xsltStylesheet *xslt, xmlDoc *indata_d, parseParams *params);
+char *sqldataExtractContent(LogContext *log, xmlNode *sql_n);
+int sqldataGetFid(LogContext *log, xmlNode *sqld, const char *fname);
+char *sqldataGetValue(LogContext *log, xmlDoc *sqld, const char *fname, int recid);
+xmlDoc *sqldataGetHostInfo(LogContext *log, xsltStylesheet *xslt, xmlDoc *summaryxml,
+ int syskey, char **hostname, char **ipaddr);
+#endif
diff --git a/server/xmlparser.xsl b/server/parser/xmlparser.xsl
index cf3483b..0313a67 100644
--- a/server/xmlparser.xsl
+++ b/server/parser/xmlparser.xsl
@@ -47,6 +47,11 @@
<xsl:text>Invalid 'syskey' parameter value: </xsl:text><xsl:value-of select="syskey"/>
</xsl:message>
</xsl:if>
+ <xsl:if test="string(number($rterid)) = 'NaN'">
+ <xsl:message terminate="yes">
+ <xsl:text>Invalid rterid' parameter value: </xsl:text><xsl:value-of select="$rterid"/>
+ </xsl:message>
+ </xsl:if>
<xsl:if test="$report_filename = ''">
<xsl:message terminate="yes">
<xsl:text>The parameter 'report_filename' parameter cannot be empty</xsl:text>
@@ -85,6 +90,29 @@
<xsl:apply-templates select="/rteval/cyclictest/RawSampleData" mode="cyclic_raw_sql"/>
</xsl:when>
+ <!-- TABLE: cyclic_histogram -->
+ <xsl:when test="$table = 'cyclic_histogram'">
+ <xsl:if test="string(number($rterid)) = 'NaN'">
+ <xsl:message terminate="yes">
+ <xsl:text>Invalid 'rterid' parameter value: </xsl:text><xsl:value-of select="$rterid"/>
+ </xsl:message>
+ </xsl:if>
+ <sqldata table="cyclic_histogram">
+ <fields>
+ <field fid="0">rterid</field>
+ <field fid="1">core</field>
+ <field fid="2">index</field>
+ <field fid="3">value</field>
+ </fields>
+ <records>
+ <xsl:apply-templates select="/rteval/cyclictest/system/histogram/bucket"
+ mode="cyclic_histogram_sql"/>
+ <xsl:apply-templates select="/rteval/cyclictest/core/histogram/bucket"
+ mode="cyclic_histogram_sql"/>
+ </records>
+ </sqldata>
+ </xsl:when>
+
<xsl:otherwise>
<xsl:message terminate="yes">
<xsl:text>Invalid 'table' parameter value: </xsl:text><xsl:value-of select="$table"/>
@@ -130,7 +158,7 @@
</xsl:template>
<xsl:template match="/rteval" mode="rtevalruns_sql">
- <sqldata table="rtevalruns" key="rterid">
+ <sqldata table="rtevalruns">
<fields>
<field fid="0">syskey</field>
<field fid="1">kernel_ver</field>
@@ -141,6 +169,8 @@
<field fid="6">load_avg</field>
<field fid="7">version</field>
<field fid="8">report_filename</field>
+ <field fid="9">rterid</field>
+ <field fid="10">submid</field>
</fields>
<records>
<record>
@@ -159,6 +189,8 @@
<value fid="6"><xsl:value-of select="loads/@load_average"/></value>
<value fid="7"><xsl:value-of select="@version"/></value>
<value fid="8"><xsl:value-of select="$report_filename"/></value>
+ <value fid="9"><xsl:value-of select="$rterid"/></value>
+ <value fid="10"><xsl:value-of select="$submid"/></value>
</record>
</records>
</sqldata>
@@ -175,7 +207,7 @@
<value fid="0"><xsl:value-of select="$rterid"/></value>
<value fid="1" type="xmlblob">
<rteval_details>
- <xsl:copy-of select="clocksource|network_config|loads|cyclictest/command_line"/>
+ <xsl:copy-of select="clocksource|services|kthreads|network_config|loads|cyclictest/command_line"/>
</rteval_details>
</value>
</record>
@@ -244,4 +276,14 @@
</records>
</sqldata>
</xsl:template>
+
+ <xsl:template match="/rteval/cyclictest/system/histogram/bucket|/rteval/cyclictest/core/histogram/bucket"
+ mode="cyclic_histogram_sql">
+ <record>
+ <value fid="0"><xsl:value-of select="$rterid"/></value>
+ <value fid="1"><xsl:value-of select="../../@id"/></value>
+ <value fid="2"><xsl:value-of select="@index"/></value>
+ <value fid="3"><xsl:value-of select="@value"/></value>
+ </record>
+ </xsl:template>
</xsl:stylesheet>
diff --git a/server/rteval_xmlrpc.py b/server/rteval_xmlrpc.py
index 334ef8c..4362d89 100644
--- a/server/rteval_xmlrpc.py
+++ b/server/rteval_xmlrpc.py
@@ -35,11 +35,10 @@ from rteval.rtevalConfig import rtevalConfig
def Dispatch(req, method, args):
# Default configuration
defcfg = {'xmlrpc_server': { 'datadir': '/var/lib/rteval',
- 'xsltpath': '/usr/share/rteval',
'db_server': 'localhost',
'db_port': 5432,
'database': 'rteval',
- 'db_username': 'xmlrpc',
+ 'db_username': 'rtevxmlrpc',
'db_password': 'rtevaldb'
}
}
diff --git a/server/rtevaldb.py b/server/rtevaldb.py
index e3c7928..d8453c9 100644
--- a/server/rtevaldb.py
+++ b/server/rtevaldb.py
@@ -27,71 +27,24 @@
import os
from database import Database
-from xmlparser import XMLSQLparser
-def register_report(config, xmldata, filename, debug=False, noaction=False):
+def register_submission(config, clientid, filename, debug=False, noaction=False):
+ "Registers a submission of a rteval report which signalises the rteval_parserd process"
+
dbc = Database(host=config.db_server, port=config.db_port, database=config.database,
user=config.db_username, password=config.db_password,
debug=debug, noaction=noaction)
- parser = XMLSQLparser(os.path.join(config.xsltpath, "xmlparser.xsl"), xmldata)
-
- systems = parser.GetSQLdata('systems')
- sysid = dbc.GetValue(systems, 0, 'sysid')
-
- # Check if system is already registered
- chk = dbc.SELECT('systems',['syskey'], where={'sysid': sysid})
- if dbc.NumTuples(chk) == 0:
- # This is a new system, register it
- res = dbc.INSERT(systems)
- if len(res) != 1:
- dbc.ROLLBACK()
- raise Exception, "** register_report(): Failed to register system [1]"
-
- syskey = res[0]
- systemhost = parser.GetSQLdata('systems_hostname', syskey=syskey)
- res = dbc.INSERT(systemhost)
- if len(res) != 1:
- dbc.ROLLBACK()
- raise Exception, "** register_report(): Failed to register system hostname/ipaddr [1]"
-
- else:
- # If this is a known system, check that hostname / IP address is the same
- syskey = dbc.GetValue(chk, 0, 0)
- systemhost = parser.GetSQLdata('systems_hostname', syskey=syskey)
- srch = {'hostname': dbc.GetValue(systemhost, 0, 'hostname'),
- 'ipaddr': dbc.GetValue(systemhost, 0, 'ipaddr')}
- chk = dbc.SELECT('systems_hostname',['hostname','ipaddr'], where=srch)
- if dbc.NumTuples(chk) == 0:
- # This is an unknown hostname, register it
- dbc.INSERT(systemhost)
+ submvars = {"table": "submissionqueue",
+ "fields": ["clientid", "filename"],
+ "records": [[clientid, filename]],
+ "returning": "submid"
+ }
- # system is now registered, including hostname and IP address, and
- # we have a reference in the 'syskey' variable.
-
- # Register rteval run
- rterun = parser.GetSQLdata('rtevalruns', syskey=syskey, report_filename=filename)
- res = dbc.INSERT(rterun)
+ res = dbc.INSERT(submvars)
if len(res) != 1:
- dbc.ROLLBACK()
- raise Exception, "** register_report(): Failed to register rteval run [1]"
- rterid = res[0] # RTEval Run ID
-
- # Register some more details about the run
- rtedet = parser.GetSQLdata('rtevalruns_details', rterid=rterid)
- dbc.INSERT(rtedet)
-
- # Register cyclic statistics data
- cyclic = parser.GetSQLdata('cyclic_statistics', rterid=rterid)
- dbc.INSERT(cyclic)
+ raise Exception("Could not register the submission")
- # Register cyclic raw data
- cycraw = parser.GetSQLdata('cyclic_rawdata', rterid=rterid)
- dbc.INSERT(cycraw)
-
- # Commit this work
dbc.COMMIT()
-
- # We're done
- return (syskey, rterid)
+ return res[0]
diff --git a/server/xmlparser.py b/server/xmlparser.py
deleted file mode 100644
index 09afd64..0000000
--- a/server/xmlparser.py
+++ /dev/null
@@ -1,146 +0,0 @@
-#
-# xmlparser.py
-# Library for parsing rteval XML files
-#
-# Copyright 2009 David Sommerseth <davids@redhat.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-#
-# For the avoidance of doubt the "preferred form" of this code is one which
-# is in an open unpatent encumbered format. Where cryptographic key signing
-# forms part of the process of creating an executable the information
-# including keys needed to generate an equivalently functional executable
-# are deemed to be part of the source code.
-#
-
-import libxml2
-import libxslt
-import hashlib
-import StringIO
-import types
-
-class XMLSQLparser(object):
- "Class for parsing XML into SQL using an XSLT template for mapping data fields"
-
- def __init__(self, xslt, xml):
- self.xml = self.__get_xml_data(xml)
-
- # Verify that this is a valid rteval XML file
- try:
- ver = float(self.xml.xpathEval('/rteval/@version')[0].content)
- if ver < 0.8:
- raise Exception, 'Unsupported rteval XML version'
- except Exception, err:
- raise Exception, 'Input file was unparsable or not a valid rteval XML file (%s)' % str(err)
-
- xsltdoc = self.__get_xml_data(xslt)
- self.parser = libxslt.parseStylesheetDoc(xsltdoc)
-
-
- def __get_xml_data(self, input):
- if hasattr(input, '__module__') and (input.__module__ == 'libxml2') and hasattr(input, 'get_type'):
- if input.get_type() == 'document_xml':
- # It's an XML document, use it directly
- return input
- elif input.get_type() == 'element':
- # It's an XML node, create a document and set node as root
- xmldoc = libxml2.newDoc("1.0")
- xmldoc.setRootElement(input)
- return xmldoc
- elif type(input) == types.StringType:
- # It's a string, assume a file name
- try:
- return libxml2.parseFile(input)
- except Exception, err:
- raise Exception, "** ERROR ** XMLSQLparser::__get_xml_data('%s') failed to load file" % str(input)
-
- # If invalid input ...
- raise AttributeError, "Unknown input type for XML/XSLT data (not a filename, xmlDoc or xmlNode)"
-
-
- def __xmlNode2string(self, node):
- doc = libxml2.newDoc('1.0')
- doc.setRootElement(node)
-
- iobuf = StringIO.StringIO()
- xmlbuf = libxml2.createOutputBuffer(iobuf, 'UTF-8')
- doc.saveFileTo(xmlbuf, 'UTF-8')
- retstr = iobuf.getvalue()
- del doc
- del xmlbuf
- del iobuf
- return retstr
-
-
- def GetSQLdata(self, tbl, rterid=None, syskey=None, report_filename=None):
- params = { 'table': '"%s"' % tbl,
- 'rterid': rterid and '"%i"' % rterid,
- 'syskey': syskey and '"%i"' % syskey,
- 'report_filename': report_filename and '"%s"' % report_filename }
- resdoc = self.parser.applyStylesheet(self.xml, params)
-
- # Extract fields, and make sure they are ordered/sorted by the fid attribute
- fields = []
- tmp_fields = {}
- for f in resdoc.xpathEval('/sqldata/fields/field'):
- tmp_fields[int(f.prop('fid'))] = f.content
-
- for f in range(0, len(tmp_fields)):
- fields.append(tmp_fields[f])
-
- # Extract values, make sure they are in the same order as the field values
- records = []
- for r in resdoc.xpathEval('/sqldata/records/record'):
- rvs = {}
- for v in r.xpathEval('value'):
- if v.prop('type') == 'xmlblob':
- fieldval = self.__xmlNode2string(v.children)
- elif v.prop('isnull') == '1':
- fieldval = None
- else:
- fieldval = v.content and v.content or None
-
- if v.hasProp('hash') and fieldval is not None:
- try:
- hash = getattr(hashlib, v.prop('hash'))
- except AttributeError:
- raise Exception, 'Unsuported hash algoritm: %s' % v.prop('hash')
-
- rvs[int(v.prop('fid'))] = hash(fieldval).hexdigest()
- else:
- rvs[int(v.prop('fid'))] = fieldval
-
- # Make sure the field values are in the correct order
- vls = []
- for v in range(0, len(rvs)):
- vls.append(rvs[v])
-
- # Append all these field values as a record
- records.append(vls)
-
- result = { 'table': resdoc.xpathEval('/sqldata/@table')[0].content,
- 'fields': fields, 'records': records}
-
- # Extract the key field being returned from INSERT statements, if set
- try:
- retkey = resdoc.xpathEval('/sqldata/@key')
- if retkey and retkey[0] and retkey[0].content:
- result['returning'] = retkey[0].content
- except:
- pass
-
- resdoc.freeDoc()
- return result
-
diff --git a/server/xmlrpc_API1.py b/server/xmlrpc_API1.py
index 95390bd..7b61234 100644
--- a/server/xmlrpc_API1.py
+++ b/server/xmlrpc_API1.py
@@ -36,7 +36,7 @@ import rtevaldb
class XMLRPC_API1():
def __init__(self, config=None, debug=False, nodbaction=False):
# Some defaults
- self.fnametrans = string.maketrans("/\\", "::") # replace path delimiters in filenames
+ self.fnametrans = string.maketrans("/\\.", "::_") # replace path delimiters in filenames
self.debug = debug
self.nodbaction = nodbaction
self.config = config
@@ -53,12 +53,12 @@ class XMLRPC_API1():
os.chdir(startdir)
- def __getfilename(self, dir, fname, comp):
+ def __getfilename(self, dir, fname, ext, comp):
idx = 0
if comp:
- filename = "%s/%s/%s.bz2" % (self.config.datadir, dir, fname.translate(self.fnametrans))
+ filename = "%s/%s/%s%s.bz2" % (self.config.datadir, dir, fname.translate(self.fnametrans), ext)
else:
- filename = "%s/%s/%s" % (self.config.datadir, dir, fname.translate(self.fnametrans))
+ filename = "%s/%s/%s%s" % (self.config.datadir, dir, fname.translate(self.fnametrans), ext)
while 1:
if not os.path.exists(filename):
@@ -90,14 +90,14 @@ class XMLRPC_API1():
# Save a copy of the report on the file system
# Make sure we have a directory to write files into
- self.__mkdatadir(self.config.datadir + '/reports/' + clientid)
- fname = self.__getfilename('reports/' + clientid,'report.xml', False)
+ self.__mkdatadir(self.config.datadir + '/queue/')
+ fname = self.__getfilename('queue/', ('%s' % clientid), '.xml', False)
xmldoc.saveFormatFileEnc(fname,'UTF-8',1)
if self.debug:
print "Copy of report: %s" % fname
- # Register the report into a database and return the rteval run id
- (syskey, rterid) = rtevaldb.register_report(self.config, xmldoc, fname,
+ # Register the submission and put it in a parse queue
+ rterid = rtevaldb.register_submission(self.config, clientid, fname,
debug=self.debug, noaction=self.nodbaction)
if self.nodbaction:
rterid = 999999999 # Fake ID when no database registration is done
@@ -116,7 +116,7 @@ class XMLRPC_API1():
self.__mkdatadir(self.datadir + '/uploads/' + clientid)
# Get a unique filename, as close as possible to the input filename
- fname = self.__getfilename('uploads/' + clientid, filename, not decompdata)
+ fname = self.__getfilename(('uploads/%s/%s' % clientid), filename, None, not decompdata)
# Save and return filename used server side
f = open(fname, "w")