diff options
author | David Sommerseth <davids@redhat.com> | 2009-10-26 14:25:41 +0100 |
---|---|---|
committer | David Sommerseth <davids@redhat.com> | 2009-10-26 14:25:41 +0100 |
commit | edeac25984921383f4b20762882690cac5470a29 (patch) | |
tree | fa79796bab0b90179f684ec9c888913122a066c5 /server | |
parent | b7ac36d9196a5559be8a67dd5bb2b164e64abbc5 (diff) | |
parent | 81c363403356852340f7b20eed8b127f79440373 (diff) | |
download | rteval-edeac25984921383f4b20762882690cac5470a29.tar.gz rteval-edeac25984921383f4b20762882690cac5470a29.tar.xz rteval-edeac25984921383f4b20762882690cac5470a29.zip |
Merge branch 'master_ipv4' into clark
Conflicts:
Makefile
rteval.spec
Diffstat (limited to 'server')
34 files changed, 4931 insertions, 238 deletions
diff --git a/server/README b/server/README.xmlrpc index f80e41e..435d679 100644 --- a/server/README +++ b/server/README.xmlrpc @@ -3,12 +3,18 @@ ** The XML-RPC server has the purpose of collecting information from -several rteval clients. Each client will get a unique system ID which -then can be used to track how each system changes behavior on -different kernels. All of the data in the summary.xml produced by the -rteval script is sent over to the XML-RPC server and a copy is saved, -default location is /var/lib/rteval. This XML file is then parsed and -the data is stored in a database for further analysis. +several rteval clients. All the data in the summary.xml produced by the +rteval script is sent over to the XML-RPC server and registered in a +submission queue. The XML-RPC server will then send back a submission +ID to the client. + +A parser daemon needs to run as well. This daemon is connected to the +same database as the XML-RPC service and it will wait for new reports in +the submission queue to be parsed. Look into the rteval/server/parser +directory for more information on setting up the rteval_parserd process. + +Each parsed report will get a unique system ID which then can be used to +track how each system changes behavior on different kernels. ** @@ -53,7 +59,7 @@ server. pg_hba.conf entry example: # TYPE DATABASE USER CIDR-ADDRESS METHOD -hostssl rteval xmlrpc 127.0.0.1/32 md5 +hostssl rteval rtevxmlrpc 127.0.0.1/32 md5 The XML-RPC database connector will always try to connect via SSL. To modify the default password, connect to the database with psql and @@ -95,7 +101,6 @@ or parameters is not set. # Paths datadir: /var/lib/rteval - xsltpath: /usr/share/rteval # Database parameters db_server: localhost diff --git a/server/gen_config.sh b/server/gen_config.sh new file mode 100755 index 0000000..4b57353 --- /dev/null +++ b/server/gen_config.sh @@ -0,0 +1,13 @@ +#/bin/sh + +APACHECONF="apache-rteval.conf" +INSTALLDIR="$1" + +echo "Creating Apache config file: apache-rteval.conf" +escinstpath="$(echo ${INSTALLDIR} | sed -e 's/\//\\\\\//g')" +expr=$(echo "s/{_INSTALLDIR_}/${escinstpath}/") +eval "sed -e ${expr} ${APACHECONF}.tpl" > ${APACHECONF} +echo "Copy the apache apache-rteval.conf into your Apache configuration" +echo "directory and restart your web server" +echo + diff --git a/server/install.sh b/server/install.sh index 0565d7a..5f67948 100755 --- a/server/install.sh +++ b/server/install.sh @@ -1,13 +1,10 @@ #!/bin/sh -PYTHON_FILES="rteval_xmlrpc.py xmlrpc_API1.py xmlparser.py rtevaldb.py database.py" -XSLT_FILES="xmlparser.xsl" +PYTHON_FILES="rteval_xmlrpc.py xmlrpc_API1.py rtevaldb.py database.py" +XSLT_FILES="parser/xmlparser.xsl" XSLTDIR="/usr/share/rteval" -APACHECONF="apache-rteval.conf" -RTEVALCONF="rteval-xmlrpc.conf" - if [ $# != 1 ]; then echo "$0 </var/www/html/.... full path to the directory the XML-RPC server will reside>" exit @@ -23,11 +20,4 @@ echo "Installing XSLT templates to ${XSLTDIR}" cp -v ${XSLT_FILES} ${XSLTDIR} echo -echo "Creating Apache config file: apache-rteval.conf" -escinstpath="$(echo ${INSTALLDIR} | sed -e 's/\//\\\\\//g')" -expr=$(echo "s/{_INSTALLDIR_}/${escinstpath}/") -eval "sed -e ${expr} ${APACHECONF}.tpl" > ${APACHECONF} -echo "Copy the apache apache-rteval.conf into your Apache configuration" -echo "directory and restart your web server" -echo - +./gen_config.sh ${INSTALLDIR} diff --git a/server/parser/Makefile b/server/parser/Makefile new file mode 100644 index 0000000..298aec0 --- /dev/null +++ b/server/parser/Makefile @@ -0,0 +1,13 @@ + +CFLAGS = -g -fPIC -I. -Wall -Werror $(shell pkg-config libxml-2.0 --cflags) $(shell pkg-config libxslt --cflags) +LDFLAGS = -lrt -lpthread -lpq $(shell pkg-config libxml-2.0 --libs) $(shell pkg-config libxslt --libs) + +OBJS = argparser.o configparser.o eurephia_nullsafe.o \ + eurephia_values.o eurephia_xml.o log.o parsethread.o pgsql.o sha1.o \ + xmlparser.o rteval_parserd.o + +rteval_parserd : $(OBJS) + gcc -g -o $@ $^ $(LDFLAGS) + +clean : + rm -f $(OBJS) rteval_parserd *~ diff --git a/server/parser/README b/server/parser/README new file mode 100644 index 0000000..793b35b --- /dev/null +++ b/server/parser/README @@ -0,0 +1,134 @@ +** +** rteval_parsed - the rteval XML report parser +** + +The purpose of the daemon is to off load the web server from the heavy duty +work of parsing and processing the rteval XML reports. The XML-RPC server +will receive the reports and put the files in a queue directory on the +filesystem and register the the submission in the database. This will notify +the rteval_parsed that a new report has been received and it will start +processing that file independently of the web/XML-RPC server. + + +** Configure rteval_parsed + +This daemon uses the same configuration file as the rest of the rteval program +suite, /etc/rteval.conf. It will parse the section named 'xmlrpc_parser'. + +The default values are: + + - xsltpath: /usr/share/rteval + Defines where it can find the xmlparser.xsl XSLT template + + - db_server: localhost + Which database server to connect to + + - db_port: 5432 + Which port to use for the database connection + + - database: rteval + Which database to make use of. + + - db_username: rtevparser + Which user name to use for the connection + + - db_password: rtevaldb_parser + Which password to use for the authentication + + - reportdir: /var/lib/rteval/report + Where to save the parsed reports + + +** rteval_parserd arguments + + -d | --daemon Run as a daemon + -l | --log <log dest> Where to put log data + -L | --log-level <verbosity> What to log + -f | --config <config file> Which configuration file to use + -t | --threads <num. threads> How many worker threads to start (def: 4) + -h | --help This help screen + +- Configuration file +By default the program will look for /etc/rteval.conf. This can be +overriden by using --config <config file>. + +- Logging +When the program is started as a daemon, it will log to syslog by default. +The default log level is 'info'. When not started as a daemon, all logging +will go to stderr by default. + +The --log argument takes either 'destination' or a file name. Unknown +destinations are treated as filenames. Valid 'destinations' are: + + stderr: - Log to stderr + stdout: - Log to stdout + syslog:[facility] - Log to syslog + <file name> - Log to given file + +For syslog the default facility is 'daemon', but can be overriden by using +one of the following facility values: + daemon, user and local0 to local7 + +Log verbosity is set by the --log-level. The valid values here are: + + emerg, emergency - Only log errors which causes the program to stop + alert - Incidents which needs immediate attention + crit, critical - Unexpected incidents which is not urgent + err, error - Parsing errors. Issues with input data + warn, warning - Incidents which may influence performance + notice - Less important warnings + info - General run information + debug - Detailed run information, incl. thread operation + +- Threads +By default, the daemon will use 5 threads. One for the main threads which +processes the submission queue and notifies the working threads. The 4 other +threads are worker threads, which will process the received reports. + +Each of the worker threads will have its own connection to the database. This +connection will be connected to the database as long as the daemon is running. +It is therefore important that you do not have more worker threads than +available database connections. + + +** POSIX Message Queue + +The daemon makes use of POSIX MQ for distributing work to the worker threads. +Each thread lives independently and polls the queue regularly for more work. +As the POSIX MQ has a pretty safe mechanism of not duplicating messages in the +implementation, no other locking facility is needed. + +On Linux, the default value for maximum messages in the queue are set to 10. +If you receive a lot of reports and the threads do not process the queue +quickly enough, it will fill up pretty quickly. If the queue is filled up, +the main thread which populates the message queue will politely go to sleep +for one minute before attempting to send new messages. To avoid this, consider +to increase the queue size by modifying /proc/sys/fs/mqueue/msg_max. + +When the daemon initialises itself, it will read this file to make sure it +uses the queue to the maximum, but not beyond that. + + +** PostgreSQL features + +The daemon depends on the PostgreSQL database. It is written with an +abstraction layer so it should, in theory, be possible to easily adopt it to +different database implementation. + +In the current implementation, it makes use of PostgreSQL's LISTEN, NOTIFY and +UNLISTEN features. A trigger is enabled on the submission queue table, which +sends a NOTIFY whenever a record is inserted into the table. The rteval_parser +daemon listens for these notifications, and will immediately poll the table +upon such a notification. + +Whenever a notification is received, it will always parse all unprocessed +reports. In addition it will also only listen for notifications when there +are no unprocessed reports. + + +** Submission queue status codes + +In the rteval database's submissionqueue table there is a status field. The +daemon will only consider records with status == 0 for processing. It do not +consider any other fields. For a better understanding of the different status +codes, look into the file statuses.h. diff --git a/server/parser/argparser.c b/server/parser/argparser.c new file mode 100644 index 0000000..e9d523a --- /dev/null +++ b/server/parser/argparser.c @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file argparser.c + * @author David Sommerseth <davids@redhat.com> + * @date Thu Oct 22 13:58:46 2009 + * + * @brief Generic argument parser + * + */ + +#include <stdio.h> +#include <stdlib.h> +#include <getopt.h> +#include <eurephia_values.h> +#include <eurephia_nullsafe.h> + + +/** + * Print a help screen to stdout + */ +void usage() { + printf("rteval_parserd: Parses new reports recieved via XML-RPC\n" + "\n" + "This program will wait for changes to the rteval 'submissionqueue' table.\n" + "When a new report is registered here, it will send this report to one of\n" + "the worker threads which will insert the parsed result into the database.\n" + "\n" + "** Program arguments:\n" + " -d | --daemon Run as a daemon\n" + " -l | --log <log dest> Where to put log data\n" + " -L | --log-level <verbosity> What to log\n" + " -f | --config <config file> Which configuration file to use\n" + " -t | --threads <num. threads> How many worker threads to start (def: 4)\n" + " -h | --help This help screen\n" + "\n" + "** Configuration file\n" + "By default the program will look for /etc/rteval.conf. This can be\n" + "overriden by using --config <config file>.\n" + "\n" + "** Logging\n" + "When the program is started as a daemon, it will log to syslog by default.\n" + "The default log level is 'info'. When not started as a daemon, all logging\n" + "will go to stderr by default.\n" + "\n" + "The --log argument takes either 'destination' or a file name. Unknown\n" + "destinations are treated as filenames. Valid 'destinations' are:\n" + "\n" + " stderr: - Log to stderr\n" + " stdout: - Log to stdout\n" + " syslog:[facility] - Log to syslog\n" + " <file name> - Log to given file\n" + "\n" + "For syslog the default facility is 'daemon', but can be overriden by using\n" + "one of the following facility values:\n" + " daemon, user and local0 to local7\n" + "\n" + "Log verbosity is set by the --log-level. The valid values here are:\n" + "\n" + " emerg, emergency - Only log errors which causes the program to stop\n" + " alert - Incidents which needs immediate attention\n" + " crit, critical - Unexpected incidents which is not urgent\n" + " err, error - Parsing errors. Issues with input data\n" + " warn, warning - Incidents which may influence performance\n" + " notice - Less important warnings\n" + " info - General run information\n" + " debug - Detailed run information, incl. thread operations\n" + "\n" + ); +} + + +/** + * Parses program arguments and puts the recognised arguments into an eurephiaVALUES struct. + * + * @param argc argument counter + * @param argv argument string table + * + * @return Returns a pointer to an eurephiaVALUES struct. On failure, the program halts. + */ +eurephiaVALUES *parse_arguments(int argc, char **argv) { + eurephiaVALUES *args = NULL; + int optidx, c; + static struct option long_opts[] = { + {"log", 1, 0, 'l'}, + {"log-level", 1, 0, 'L'}, + {"config", 1, 0, 'f'}, + {"threads", 1, 0, 't'}, + {"daemon", 0, 0, 'd'}, + {"help", 0, 0, 'h'}, + {0, 0, 0, 0} + }; + + args = eCreate_value_space(NULL, 21); + eAdd_value(args, "daemon", "0"); + eAdd_value(args, "configfile", "/etc/rteval.conf"); + eAdd_value(args, "threads", "4"); + + while( 1 ) { + optidx = 0; + c = getopt_long(argc, argv, "l:L:f:t:dh", long_opts, &optidx); + if( c == -1 ) { + break; + } + + switch( c ) { + case 'l': + eUpdate_value(args, "log", optarg, 1); + break; + case 'L': + eUpdate_value(args, "loglevel", optarg, 1); + break; + case 'f': + eUpdate_value(args, "configfile", optarg, 0); + break; + case 't': + eUpdate_value(args, "threads", optarg, 0); + break; + case 'd': + eUpdate_value(args, "daemon", "1", 0); + break; + case 'h': + usage(); + exit(0); + } + } + + // If logging is not configured, and it is not run as a daemon + // -> log to stderr: + if( (eGet_value(args, "log") == NULL) + && (atoi_nullsafe(eGet_value(args, "daemon")) == 0) ) { + eAdd_value(args, "log", "stderr:"); + } + + return args; +} diff --git a/server/parser/argparser.h b/server/parser/argparser.h new file mode 100644 index 0000000..53214e1 --- /dev/null +++ b/server/parser/argparser.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file argparser.h + * @author David Sommerseth <davids@redhat.com> + * @date Thu Oct 22 13:58:46 2009 + * + * @brief Generic argument parser + * + */ + +#ifndef _RTEVAL_ARGPARSER_h +#define _RTEVAL_ARGPARSER_h + +eurephiaVALUES *parse_arguments(int argc, char **argv); + +#endif diff --git a/server/parser/configparser.c b/server/parser/configparser.c new file mode 100644 index 0000000..1e68d23 --- /dev/null +++ b/server/parser/configparser.c @@ -0,0 +1,182 @@ +/* configparser.c - Read and parse config files + * + * This code is based on the fragments from the eurephia project. + * + * GPLv2 Copyright (C) 2009 + * David Sommerseth <davids@redhat.com> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +/** + * @file configparser.c + * @author David Sommerseth <davids@redhat.com> + * @date 2009-10-01 + * + * @brief Config file parser + * + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <assert.h> + +#include <eurephia_nullsafe.h> +#include <eurephia_values.h> +#include <configparser.h> +#include <log.h> + +/** + * Parse one single configuration line into a eurephiaVALUES key/value pair. It will also ignore + * comment lines, and also remove the comments on the line of the configuration line so that only + * the key/value information is extracted. + * + * @param line Input configuration line + * + * @return eurephiaVALUES pointer containing the parsed result. On error or if no valid config + * line was found, NULL is returned. + */ +static inline eurephiaVALUES *parse_config_line(LogContext *log, const char *line) { + char *cp = NULL, *key = NULL, *val = NULL, *ptr = NULL;; + eurephiaVALUES *ret = NULL; + + if( *line == '#' ) { + return NULL; + } + + cp = strdup(line); + key = cp; + val = strpbrk(cp, "=:"); + if( val == NULL ) { + free_nullsafe(cp); + return NULL; + } + *val = '\0'; val++; + + // Discard comments at the end of a line + if( (ptr = strpbrk(val, "#")) != NULL ) { + *ptr = '\0'; + } + + // Left trim + while( ((*key == 0x20) || (*key == 0x0A) || (*key == 0x0D)) ) { + key++; + } + while( ((*val == 0x20) || (*val == 0x0A) || (*val == 0x0D)) ) { + val++; + } + + // Right trim + ptr = key + strlen_nullsafe(key) - 1; + while( ((*ptr == 0x20) || (*ptr == 0x0A) || (*ptr == 0x0D)) && (ptr > key) ) { + ptr--; + } + ptr++; + *ptr = '\0'; + + ptr = val + strlen_nullsafe(val) - 1; + while( ((*ptr == 0x20) || (*ptr == 0x0A) || (*ptr == 0x0D)) && (ptr > val) ) { + ptr--; + } + ptr++; + *ptr = '\0'; + + // Put key/value into a eurephiaVALUES struct and return it + ret = eCreate_value_space(log, 20); + ret->key = strdup(key); + ret->val = strdup(val); + + free_nullsafe(cp); + return ret; +} + + +static inline eurephiaVALUES *default_cfg_values(LogContext *log, eurephiaVALUES *prgargs) { + eurephiaVALUES *cfg = NULL, *ptr = NULL; + + cfg = eCreate_value_space(log, 20); + eAdd_value(cfg, "datadir", "/var/lib/rteval"); + eAdd_value(cfg, "xsltpath", "/usr/share/rteval"); + eAdd_value(cfg, "db_server", "localhost"); + eAdd_value(cfg, "db_port", "5432"); + eAdd_value(cfg, "database", "rteval"); + eAdd_value(cfg, "db_username", "rtevparser"); + eAdd_value(cfg, "db_password", "rtevaldb_parser"); + eAdd_value(cfg, "reportdir", "/var/lib/rteval/reports"); + + // Copy over the arguments to the config, update existing settings + for( ptr = prgargs; ptr; ptr = ptr->next ) { + eUpdate_value(cfg, ptr->key, ptr->val, 1); + } + + return cfg; +} + +/** + * Parses a section of a config file and puts it into an eurephiaVALUES key/value stack + * + * @param log Initialised log context + * @param prgargs Parsed command line arguments (see parse_arguments()) + * @param section Section to read from the config file + * + * @return Returns a pointer to an eurephiaVALUES stack containing the configuration on success, + * otherwise NULL. + */ +eurephiaVALUES *read_config(LogContext *log, eurephiaVALUES *prgargs, const char *section) { + FILE *fp = NULL; + char *buf = NULL, *sectmatch = NULL, *cfgname = NULL; + int sectfound = 0; + eurephiaVALUES *cfg = NULL; + struct stat fi; + + cfgname = eGet_value(prgargs, "configfile"); + if( stat(cfgname, &fi) == -1 ) { + writelog(log, LOG_EMERG, "Could not open the config file: %s", cfgname); + return NULL; + } + + if( (fp = fopen(cfgname, "r")) == NULL ) { + writelog(log, LOG_EMERG, "Could not open the config file: %s", cfgname); + return NULL; + } + + buf = (char *) malloc_nullsafe(log, fi.st_size+2); + sectmatch = (char *) malloc_nullsafe(log, strlen_nullsafe(section)+4); + sprintf(sectmatch, "[%s]", section); + + cfg = default_cfg_values(log, prgargs); + writelog(log, LOG_DEBUG, "Reading config file: %s", cfgname); + while( fgets(buf, fi.st_size, fp) != NULL ) { + if( strncmp(buf, "[", 1) == 0 ) { + sectfound = (!sectfound && (strncmp(buf, sectmatch, strlen(sectmatch)) == 0)); + continue; + } + + if( sectfound ) { + eurephiaVALUES *prm = parse_config_line(log, buf); + if( prm != NULL ) { + cfg = eUpdate_valuestruct(cfg, prm, 1); + } + } + }; + free_nullsafe(buf); + free_nullsafe(sectmatch); + fclose(fp); fp = NULL; + + return cfg; +} diff --git a/server/parser/configparser.h b/server/parser/configparser.h new file mode 100644 index 0000000..54a904b --- /dev/null +++ b/server/parser/configparser.h @@ -0,0 +1,38 @@ +/* configparser.h - Read and parse config files + * + * This code is based on the fragments from the eurephia project. + * + * GPLv2 Copyright (C) 2009 + * David Sommerseth <davids@redhat.com> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +/** + * @file configparser.h + * @author David Sommerseth <davids@redhat.com> + * @date 2009-10-01 + * + * @brief Config file parser + * + */ + +#ifndef _CONFIGPARSER_H +#define _CONFIGPARSER_H + +eurephiaVALUES *read_config(LogContext *log, eurephiaVALUES *prgargs, const char *section); + +#endif diff --git a/server/parser/eurephia_nullsafe.c b/server/parser/eurephia_nullsafe.c new file mode 100644 index 0000000..e6dad0e --- /dev/null +++ b/server/parser/eurephia_nullsafe.c @@ -0,0 +1,67 @@ +/* eurephia_nullsafe.c + * + * standard C string functions, which is made NULL safe by checking + * if input value is NULL before performing the action. + * + * This version is modified to work outside the eurephia project. + * + * GPLv2 only - Copyright (C) 2009 + * David Sommerseth <dazo@users.sourceforge.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +/** + * @file eurephia_nullsafe.c + * @author David Sommerseth <dazo@users.sourceforge.net> + * @date 2009-09-07 + * + * @brief standard C string functions, which is made NULL safe by checking + * if input value is NULL before performing the action. + * + */ + +#include <stdio.h> +#include <stdlib.h> + +#include <log.h> + +#if __GNUC__ >= 3 +#define __malloc__ __attribute__((malloc)) +#else /* If not GCC 3 or newer, disable optimisations */ +#define __malloc__ +#endif + +/** + * This replaces the use of malloc() and memset(). This function uses calloc + * internally, which results in the memory region being zero'd by the kernel + * on memory allocation. + * + * @param log Log context + * @param sz size of the memory region being allocated + * + * @return Returns a void pointer to the memory region on success, otherwise NULL + */ +__malloc__ void *malloc_nullsafe(LogContext *log, size_t sz) { + void *buf = NULL; + + buf = calloc(1, sz); /* Using calloc, also gives a zero'd memory region */ + if( !buf ) { + writelog(log, LOG_EMERG, "Could not allocate memory region for %ld bytes", sz); + exit(9); + } + return buf; +} diff --git a/server/parser/eurephia_nullsafe.h b/server/parser/eurephia_nullsafe.h new file mode 100644 index 0000000..e0b2f32 --- /dev/null +++ b/server/parser/eurephia_nullsafe.h @@ -0,0 +1,116 @@ +/* eurephia_nullsafe.h + * + * standard C string functions, which is made NULL safe by checking + * if input value is NULL before performing the action. + * + * This version is modified to work outside the eurephia project. + * + * GPLv2 only - Copyright (C) 2008, 2009 + * David Sommerseth <dazo@users.sourceforge.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +/** + * @file eurephia_nullsafe.h + * @author David Sommerseth <dazo@users.sourceforge.net> + * @date 2008-08-06 + * + * @brief standard C string functions, which is made NULL safe by checking + * if input value is NULL before performing the action. + * + */ + +#ifndef EUREPHIA_NULLSAFE_H_ +#define EUREPHIA_NULLSAFE_H_ + +#include <log.h> + +/** + * atoi() wrapper. Converts any string into a integer + * + * @param str Input string + * + * @return Returns integer + */ +#define atoi_nullsafe(str) (str != NULL ? atoi(str) : 0) + + +/** + * strdup() wrapper. Duplicates the input string. + * + * @param str Input string to be duplicated + * + * @return Returns a pointer to the duplicate (char *) on success, NULL otherwise. + * If input was NULL, NULL is returned. + */ +#define strdup_nullsafe(str) (str != NULL ? strdup(str) : NULL) + + +/** + * Wrapper macro, which appends a string to a destination string without exceeding the size + * of the destination buffer. + * + * @param dest Pointer to the destination buffer + * @param src Pointer to the value being concatenated to the destination string. + * @param size Size of the destination buffer + */ +#define append_str(dest, src, size) strncat(dest, src, (size - strlen_nullsafe(dest))) + + +/** + * strlen() wrapper. Returns the length of a string + * + * @param str Input string + * + * @return Returns int with length of string. If input is NULL, it returns 0. + */ +#define strlen_nullsafe(str) (str != NULL ? strlen(str) : 0) + + +void *malloc_nullsafe(LogContext *, size_t); + +/** + * Null safe free(). It will not attempt to free a pointer which is NULL. + * + * @param ptr Pointer to the memory region being freed. + * + */ +#define free_nullsafe(ptr) if( ptr ) { free(ptr); ptr = NULL; } + + +/** + * Function which will return a default string value if no input data was provided. + * + * @param str Input string + * @param defstr Default string + * + * @return Returns the pointer to the input string if the string length > 0. Otherwise it + * will return a pointer to the default string. + */ +#define defaultValue(str, defstr) (strlen_nullsafe(str) == 0 ? defstr : str) + + +/** + * Function which will return a default integer value if no input data was provided. + * + * @param ival input integer value + * @param defval default integer value + * + * @return Returns the ival value if it is > 0, otherwise defval value is returned. + */ +#define defaultIntValue(ival, defval) (ival == 0 ? defval : ival) +#endif /* !EUREPHIA_NULLSAFE_H_ */ diff --git a/server/parser/eurephia_values.c b/server/parser/eurephia_values.c new file mode 100644 index 0000000..dbdbc13 --- /dev/null +++ b/server/parser/eurephia_values.c @@ -0,0 +1,319 @@ +/* eurephia_values.c -- Generic interface for processing key->value pairs + * + * This version is modified to work outside the eurephia project. + * + * GPLv2 only - Copyright (C) 2008 + * David Sommerseth <dazo@users.sourceforge.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * @file eurephia_values.c + * @author David Sommerseth <dazo@users.sourceforge.net> + * @date 2008-08-06 + * + * @brief Generic interface for handling key->value pairs + * + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include <eurephia_nullsafe.h> +#include <eurephia_values_struct.h> + +/** + * Internal function. Makes sure a eurephiaVALUES stack is freed up + * + * @param vls Pointer to a eurephiaVALUES stack. + */ +static inline void do_free_vals(eurephiaVALUES *vls) { + if( vls->next != NULL ) { + do_free_vals(vls->next); + } + free_nullsafe(vls->key); + free_nullsafe(vls->val); + free_nullsafe(vls); +} + + +/** + * Function for freeing up an eurephiaVALUES stack. This function is normally not called + * directly, but usually via the eFree_values(...) macro. + * + * @param vls Pointer to a eurephiaVALUES stack to be freed. + */ +void eFree_values_func(eurephiaVALUES *vls) { + if( (vls == NULL) ) { + return; + } + do_free_vals(vls); +} + + +/** + * Retrieve an eurephiaVALUES element for a given value key + * + * @param vls Pointer to the eurephiaVALUES stack where to search for the element + * @param key String containing the key name of the value requested. + * + * @return Returns an eurephiaVALUES element on success, otherwise NULL. + */ +eurephiaVALUES *eGet_valuestruct(eurephiaVALUES *vls, const char *key) +{ + eurephiaVALUES *ptr = NULL; + + if( (vls == NULL) || (key == NULL) ) { + return NULL; + } + + ptr = vls; + while( ptr != NULL ) { + if( (ptr->key != NULL) && (strcmp(key, ptr->key) == 0) ) { + return ptr; + } + ptr = ptr->next; + } + return NULL; +} + + +/** + * Retrieves the value of a given key from an eurephiaVALUES stack. + * + * @param vls Pointer to an eurephiaVALUES stack where to search for the value + * @param key String containing the key name of the value requested + * + * @return Returns a string (char *) with the requested value if found, otherwise NULL. + */ +char *eGet_value(eurephiaVALUES *vls, const char *key) +{ + eurephiaVALUES *ptr = NULL; + + ptr = eGet_valuestruct(vls, key); + return (ptr != NULL ? ptr->val : NULL); +} + + +/** + * Creates a new eurephiaVALUES stack + * + * @param log Log context + * @param evgid int value, giving the stack an ID number. Useful when looking through log files later on. + * + * @return Returns an empty eurephiaVALUES struct on success, otherwise NULL. + */ +eurephiaVALUES *eCreate_value_space(LogContext *log, int evgid) +{ + eurephiaVALUES *ptr = NULL; + + ptr = (eurephiaVALUES *) malloc_nullsafe(log, sizeof(eurephiaVALUES) + 2); + ptr->log = log; + ptr->evgid = evgid; + return ptr; +} + + +/** + * Adds a new eurephiaVALUES stack to another eurephiaVALUES stack. If the evgid value differs, it will + * be overwritten with the value of the destination stack. + * + * @param vls Destination eurephiaVALUES stack + * @param newval Source eurephiaVALUES stack + */ +void eAdd_valuestruct(eurephiaVALUES *vls, eurephiaVALUES *newval) { + eurephiaVALUES *ptr = NULL; + int vid = 0; + + assert(vls != NULL); + + if( (vls->key == NULL) && (vls->val == NULL) && (vls->next == NULL) && (vls->evid == 0)) { + // Update header record if it is empty, by copying newval record. Free newval afterwards + vls->key = strdup(newval->key); + vls->val = strdup(newval->val); + vls->evid = 0; + vls->next = NULL; + do_free_vals(newval); + } else { + // Add values to the value chain, loop to the end and append it + ptr = vls; + while( ptr->next != NULL ) { + ptr = ptr->next; + vid = (vid > ptr->evid ? vid : ptr->evid); + } + newval->evid = vid+1; // Increase the value counter + newval->evgid = ptr->evgid; + ptr->next = newval; + } +} + + +/** + * Adds a new key/value pair to an eurephiaVALUES stack + * + * @param vls Destination eurephiaVALUES stack + * @param key Key name for the value being stored + * @param val Value to be stored + */ +void eAdd_value(eurephiaVALUES *vls, const char *key, const char *val) +{ + eurephiaVALUES *ptr = NULL; + + assert(vls != NULL); + + // Allocate buffer and save values + ptr = eCreate_value_space(vls->log, vls->evid); + if( ptr == NULL ) { + writelog(vls->log, LOG_EMERG, "Failed to add value to the value chain"); + exit(9); + } + ptr->key = strdup_nullsafe(key); + ptr->val = strdup_nullsafe(val); + ptr->evgid = vls->evgid; + + // Add value struct to the chain + eAdd_valuestruct(vls, ptr); +} + + +/** + * Updates the value of a key in a values stack + * + * @param vls eurephiaVALUES key/value stack to update + * @param key String with key name to update + * @param newval String with the new value + * @param addunkn Add unknown keys. If set to 1, if the key is not found it will add a new key + */ +void eUpdate_value(eurephiaVALUES *vls, const char *key, const char *newval, const int addunkn) { + eurephiaVALUES *ptr = NULL; + + assert( (vls != NULL) && (key != NULL) ); + + ptr = eGet_valuestruct(vls, key); + if( ptr ) { + free_nullsafe(ptr->val); + ptr->val = strdup_nullsafe(newval); + } else if( addunkn == 1 ) { + eAdd_value(vls, key, newval); + } +} + + +/** + * Updates a value struct element based on another value struct element contents (key/value) + * + * @param vls eurephiaVALUES key/value stack to update + * @param newval eurephiaVALUES element with the new value + * @param addunkn Add unknown keys. If set to 1, if the key is not found it will add a new key + * + * @return Returns a pointer to the first element in the chain. If the element being updated + * was the first element in the old chain, the first element will be a new element with a + * new address. + */ +eurephiaVALUES *eUpdate_valuestruct(eurephiaVALUES *vls, eurephiaVALUES *newval, const int addunkn) { + eurephiaVALUES *ptr = NULL, *prevptr = NULL; + + assert( (vls != NULL) && (newval != NULL) && (newval->key != NULL) ); + + prevptr = vls; + for( ptr = vls; ptr != NULL; ptr = ptr->next ) { + if( (ptr->key != NULL) && (strcmp(newval->key, ptr->key) == 0) ) { + newval->evgid = ptr->evgid; + newval->evid = ptr->evid; + newval->next = ptr->next; + ptr->next = NULL; + if( ptr == vls ) { + // If the element found is the first one, do special treatment + eFree_values_func(ptr); + return newval; + } else { + prevptr->next = newval; + eFree_values_func(ptr); + return vls; + } + } + prevptr = ptr; + } + + if( addunkn == 1 ) { + eAdd_valuestruct(vls, newval); + } + return vls; +} + + +/** + * Removes the key/value pair identified by evgid and evid from the given eurephiaVALUES chain + * + * @param vls Pointer to an eurephiaVALUES chain with the data + * @param evgid Group ID of the chain + * @param evid Element ID of the chain element to be removed + * + * @return Returns a pointer to the chain. The pointer is only changed if the first element in the + * chain is deleted + */ +eurephiaVALUES *eRemove_value(eurephiaVALUES *vls, unsigned int evgid, unsigned int evid) { + eurephiaVALUES *ptr = NULL, *prev_ptr = NULL; + int found = 0; + + // Find the value element + for( ptr = vls; ptr != NULL; ptr = ptr->next ) { + if( (ptr->evgid == evgid) && (ptr->evid == evid) ) { + found = 1; + break; + } + prev_ptr = ptr; + } + + if( !found ) { + return vls; + } + + if( ptr != vls ) { + prev_ptr->next = ptr->next; + ptr->next = NULL; + eFree_values_func(ptr); + return vls; + } else { + prev_ptr = ptr->next; + ptr->next = NULL; + eFree_values_func(ptr); + return prev_ptr; + } +} + + +/** + * Counts number of elements in an eurephiaVALUES chain. + * + * @param vls eurephiaVALUES pointer to be counted + * + * @return Returns number of elements found. + */ +unsigned int eCount(eurephiaVALUES *vls) { + eurephiaVALUES *ptr = NULL; + unsigned int c = 0; + + if( vls == NULL ) { + return 0; + } + for(ptr = vls; ptr != NULL; ptr = ptr->next ) { + c++; + } + return c; +} diff --git a/server/parser/eurephia_values.h b/server/parser/eurephia_values.h new file mode 100644 index 0000000..c63eb4c --- /dev/null +++ b/server/parser/eurephia_values.h @@ -0,0 +1,61 @@ +/* eurephia_values.h -- Generic interface for processing key->value pairs + * + * This version is modified to work outside the eurephia project. + * + * GPLv2 only - Copyright (C) 2008 + * David Sommerseth <dazo@users.sourceforge.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +/** + * @file eurephia_values.h + * @author David Sommerseth <dazo@users.sourceforge.net> + * @date 2008-08-06 + * + * @brief Generic interface for handling key->value pairs + * + */ + +#include <eurephia_values_struct.h> + +#ifndef EUREPHIA_VALUES_H_ +#define EUREPHIA_VALUES_H_ + + +eurephiaVALUES *eGet_valuestruct(eurephiaVALUES *vls, const char *key); +char *eGet_value(eurephiaVALUES *vls, const char *key); + +eurephiaVALUES *eCreate_value_space(LogContext *log, int evid); + +void eAdd_valuestruct(eurephiaVALUES *vls, eurephiaVALUES *newval); +void eAdd_value(eurephiaVALUES *vls, const char *key, const char *val); +void eUpdate_value(eurephiaVALUES *vls, const char *key, const char *newval, const int addunkn); +eurephiaVALUES *eUpdate_valuestruct(eurephiaVALUES *vls, eurephiaVALUES *newval, const int addunkn); +eurephiaVALUES *eRemove_value(eurephiaVALUES *vls, unsigned int evgid, unsigned int evid); +unsigned int eCount(eurephiaVALUES *vls); + +/** + * Front-end function for eFree_values_func(). Frees eurephiaVALUES pointer chain and + * sets the pointer to NULL. + * + * @param v eurephiaVALUES pointer which is being freed. + * + */ +#define eFree_values(v) { eFree_values_func(v); v = NULL; } +void eFree_values_func(eurephiaVALUES *vls); + +#endif /* !EUREPHIA_VALUES_H_ */ diff --git a/server/parser/eurephia_values_struct.h b/server/parser/eurephia_values_struct.h new file mode 100644 index 0000000..7eb8b74 --- /dev/null +++ b/server/parser/eurephia_values_struct.h @@ -0,0 +1,51 @@ +/* eurephia_values.h -- eurephiaVALUES struct typedef + * + * GPLv2 only - Copyright (C) 2008 + * David Sommerseth <dazo@users.sourceforge.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +/** + * @file eurephia_values_struct.h + * @author David Sommerseth <dazo@users.sourceforge.net> + * @date 2008-11-05 + * + * @brief Definition of the eurephiaVALUES struct + * + */ + +#ifndef EUREPHIA_VALUES_STRUCT_H_ +# define EUREPHIA_VALUES_STRUCT_H_ + +#include <log.h> + +/** + * eurephiaVALUES is a pointer chain with key/value pairs. If having several + * such pointer chains, they can be given different group IDs to separate them, + * which is especially useful during debugging. + * + */ +typedef struct __eurephiaVALUES { + LogContext *log; /**< Pointer to an established log context, used for logging */ + unsigned int evgid; /**< Group ID, all elements in the same chain should have the same value */ + unsigned int evid; /**< Unique ID per element in a pointer chain */ + char *key; /**< The key name of a value */ + char *val; /**< The value itself */ + struct __eurephiaVALUES *next; /**< Pointer to the next element in the chain. NULL == end of chain */ +} eurephiaVALUES; + +#endif /* !EUREPHIA_VALUES_STRUCT_H_ */ diff --git a/server/parser/eurephia_xml.c b/server/parser/eurephia_xml.c new file mode 100644 index 0000000..c8c389e --- /dev/null +++ b/server/parser/eurephia_xml.c @@ -0,0 +1,160 @@ +/* eurephia_xml.c -- Generic helper functions for XML parsing + * + * This version is modified to work outside the eurephia project. + * + * GPLv2 only - Copyright (C) 2008, 2009 + * David Sommerseth <dazo@users.sourceforge.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +/** + * @file eurephia_xml.c + * @author David Sommerseth <dazo@users.sourceforge.net> + * @date 2008-12-15 + * + * @brief Generic XML parser functions + * + * + */ + +#include <stdarg.h> +#include <string.h> +#include <assert.h> + +#include <libxml/tree.h> +#include <libxml/xmlsave.h> +#include <libxml/xmlstring.h> + +#include <eurephia_nullsafe.h> + + +/** + * Retrieves a given XML node attribute/property + * + * @param attr xmlAttr pointer from an xmlNode pointer. + * @param key The attribute name to search for + * + * @return The value of the found attribute. If not found, NULL is returned. + */ +char *xmlGetAttrValue(xmlAttr *attr, const char *key) { + xmlAttr *aptr; + xmlChar *x_key = NULL; + + x_key = xmlCharStrdup(key); + assert( x_key != NULL ); + + for( aptr = attr; aptr != NULL; aptr = aptr->next ) { + if( xmlStrcmp(aptr->name, x_key) == 0 ) { + free_nullsafe(x_key); + return (char *)(aptr->children != NULL ? aptr->children->content : NULL); + } + } + free_nullsafe(x_key); + return NULL; +} + + +/** + * Loops through a xmlNode chain to look for a given tag. The search is not recursive. + * + * @param node xmlNode pointer where to look + * @param key the name of the XML tag to find + * + * @return xmlNode pointer to the found xmlNode. NULL is returned if not found. + */ +xmlNode *xmlFindNode(xmlNode *node, const char *key) { + xmlNode *nptr = NULL; + xmlChar *x_key = NULL; + + if( (node == NULL) || (node->children == NULL) ) { + return NULL; + } + + x_key = xmlCharStrdup(key); + assert( x_key != NULL ); + + for( nptr = node->children; nptr != NULL; nptr = nptr->next ) { + if( xmlStrcmp(nptr->name, x_key) == 0 ) { + free_nullsafe(x_key); + return nptr; + } + } + free_nullsafe(x_key); + return NULL; +} + + +/** + * Return the text content of a given xmlNode + * + * @param n xmlNode to extract the value from. + * + * @return returns a char pointer with the text contents of an xmlNode. + */ +inline char *xmlExtractContent(xmlNode *n) { + return (char *) (((n != NULL) && (n->children != NULL)) ? n->children->content : NULL); +} + + +/** + * Get the text contents of a given xmlNode + * + * @param node An xmlNode pointer where to look for the contents + * @param key Name of the tag to retrieve the content of. + * + * @return Returns a string with the text content, if the node is found. Otherwise, NULL is returned. + */ +inline char *xmlGetNodeContent(xmlNode *node, const char *key) { + return xmlExtractContent(xmlFindNode(node, key)); +} + + +/** + * Serialises an xmlNode to a string + * + * @param log Log context + * @param node Input XML node to be serialised + * + * @return Returns a pointer to a new buffer containing the serialised data. This buffer must be freed + * after usage + */ +char *xmlNodeToString(LogContext *log, xmlNode *node) { + xmlBuffer *buf = NULL; + xmlSaveCtxt *serctx = NULL; + char *ret = NULL; + + if( node == NULL ) { + writelog(log, LOG_ALERT, "xmlNodeToString: Input data is NULL"); + return NULL; + } + + buf = xmlBufferCreate(); + assert( buf != NULL ); + + serctx = xmlSaveToBuffer(buf, "UTF-8", XML_SAVE_NO_EMPTY|XML_SAVE_NO_DECL); + assert( serctx != NULL ); + + if( xmlSaveTree(serctx, node) < 0 ) { + writelog(log, LOG_ALERT, "xmlNodeToString: Failed to serialise xmlNode"); + return NULL; + } + xmlSaveClose(serctx); + + ret = strdup_nullsafe((char *) xmlBufferContent(buf)); + xmlBufferFree(buf); + return ret; +} diff --git a/server/parser/eurephia_xml.h b/server/parser/eurephia_xml.h new file mode 100644 index 0000000..dea72eb --- /dev/null +++ b/server/parser/eurephia_xml.h @@ -0,0 +1,56 @@ +/* eurephia_xml.h -- Generic helper functions for XML parsing + * + * This version is modified to work outside the eurephia project. + * + * GPLv2 only - Copyright (C) 2008 + * David Sommerseth <dazo@users.sourceforge.net> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +/** + * @file eurephia_xml.h + * @author David Sommerseth <dazo@users.sourceforge.net> + * @date 2008-12-15 + * + * @brief Generic XML parser functions + * + */ + + +#ifndef EUREPHIA_XML_H_ +#define EUREPHIA_XML_H_ + +#include <stdarg.h> + +#include <libxml/tree.h> + +/** + * Simple iterator macro for iterating xmlNode pointers + * + * @param start Pointer to an xmlNode where to start iterating + * @param itn An xmlNode pointer which will be used for the iteration. + */ +#define foreach_xmlnode(start, itn) for( itn = start; itn != NULL; itn = itn->next ) + +char *xmlGetAttrValue(xmlAttr *properties, const char *key); +xmlNode *xmlFindNode(xmlNode *node, const char *key); + +inline char *xmlExtractContent(xmlNode *n); +inline char *xmlGetNodeContent(xmlNode *node, const char *key); +char *xmlNodeToString(LogContext *log, xmlNode *node); + +#endif /* !EUREPHIA_XML_H_ */ diff --git a/server/parser/log.c b/server/parser/log.c new file mode 100644 index 0000000..b7c2ad8 --- /dev/null +++ b/server/parser/log.c @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file log.c + * @author David Sommerseth <davids@redhat.com> + * @date Wed Oct 21 11:38:51 2009 + * + * @brief Generic log functions + * + */ + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <errno.h> +#include <assert.h> +#include <stdarg.h> +#include <pthread.h> +#include <syslog.h> + +#include <eurephia_nullsafe.h> +#include <log.h> + +/** + * Maps defined log level strings into syslog + * compatible LOG_* integer values + */ +static struct { + const char *priority_str; + const int prio_level; +} syslog_prio_map[] = { + {"emerg", LOG_EMERG}, + {"emergency", LOG_EMERG}, + {"alert", LOG_ALERT}, + {"crit", LOG_CRIT}, + {"critical", LOG_CRIT}, + {"err", LOG_ERR}, + {"error", LOG_ERR}, + {"warning", LOG_WARNING}, + {"warn", LOG_WARNING}, + {"notice", LOG_NOTICE}, + {"info", LOG_INFO}, + {"debug", LOG_DEBUG}, + {NULL, 0} +}; + + +/** + * Initialises a log context. It parses the log destination and log level and + * prepares a context which can be used by writelog() + * + * @param logdest String containing either syslog:[facility], stderr: or stdout:, or a file name. + * @param loglvl Defines the log level. Can be one of the values defined in syslog_prio_map. + * + * @return Returns a pointer to a log context on success, otherwise NULL. + */ +LogContext *init_log(const char *logdest, const char *loglvl) { + LogContext *logctx = NULL; + int i; + + logctx = (LogContext *) calloc(1, sizeof(LogContext)+2); + assert( logctx != NULL); + + logctx->logfp = NULL; + + // Get the int value of the log level string + logctx->verbosity = -1; + if( loglvl ) { + for( i = 0; syslog_prio_map[i].priority_str; i++ ) { + if( strcasecmp(loglvl, syslog_prio_map[i].priority_str) == 0 ) { + logctx->verbosity = syslog_prio_map[i].prio_level; + break; + } + } + } + + // If log level is not set, set LOG_INFo as default + if( logctx->verbosity == -1 ) { + logctx->verbosity = LOG_INFO; + } + + if( logdest == NULL ) { + logctx->logtype = ltSYSLOG; + openlog("rteval_parserd", LOG_PID, LOG_DAEMON); + } else { + if( strncmp(logdest, "syslog:", 7) == 0 ) { + const char *fac = logdest+7; + int facid = LOG_DAEMON; + + if( strcasecmp(fac, "local0") == 0 ) { + facid = LOG_LOCAL0; + } else if( strcasecmp(fac, "local1") == 0 ) { + facid = LOG_LOCAL1; + } else if( strcasecmp(fac, "local2") == 0 ) { + facid = LOG_LOCAL2; + } else if( strcasecmp(fac, "local3") == 0 ) { + facid = LOG_LOCAL3; + } else if( strcasecmp(fac, "local4") == 0 ) { + facid = LOG_LOCAL4; + } else if( strcasecmp(fac, "local5") == 0 ) { + facid = LOG_LOCAL5; + } else if( strcasecmp(fac, "local6") == 0 ) { + facid = LOG_LOCAL6; + } else if( strcasecmp(fac, "local7") == 0 ) { + facid = LOG_LOCAL7; + } else if( strcasecmp(fac, "user") == 0 ) { + facid = LOG_USER; + } + logctx->logtype = ltSYSLOG; + openlog("rteval_parserd", LOG_PID, facid); + } else if( strcmp(logdest, "stderr:") == 0 ) { + logctx->logtype = ltCONSOLE; + logctx->logfp = stderr; + } else if( strcmp(logdest, "stdout:") == 0 ) { + logctx->logtype = ltCONSOLE; + logctx->logfp = stdout; + } else { + logctx->logtype = ltFILE; + logctx->logfp = fopen(logdest, "a"); + if( logctx->logfp == NULL ) { + fprintf(stderr, "** ERROR ** Failed to open log file %s: %s\n", + logdest, strerror(errno)); + free_nullsafe(logctx); + return NULL; + } + } + } + + if( logctx->logtype != ltSYSLOG ) { + static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; + logctx->mtx_log = &mtx; + } + return logctx; +} + + +/** + * Tears down a log context. Closes log files and releases memory used by the log context. + * + * @param lctx Log context to close + */ +void close_log(LogContext *lctx) { + if( !lctx ) { + return; + } + + switch( lctx->logtype ) { + case ltFILE: + fclose(lctx->logfp); + break; + + case ltSYSLOG: + closelog(); + break; + + case ltCONSOLE: + break; + } + free_nullsafe(lctx); +} + + +/** + * Write data to the log. + * + * @param lctx Log context, where the data will be logged + * @param loglvl Log level. See the priorities for syslog(3) for valid values. + * @param fmt Data to be logged (stdarg) + */ +void writelog(LogContext *lctx, unsigned int loglvl, const char *fmt, ... ) { + if( !lctx || !fmt ) { + return; + } + + if( lctx->verbosity >= loglvl ) { + va_list ap; + + va_start(ap, fmt); + switch( lctx->logtype ) { + case ltSYSLOG: + vsyslog(loglvl, fmt, ap); + break; + + case ltCONSOLE: + case ltFILE: + pthread_mutex_lock(lctx->mtx_log); + switch( loglvl ) { + case LOG_EMERG: + fprintf(lctx->logfp, "** EMERG ERROR ** "); + break; + case LOG_ALERT: + fprintf(lctx->logfp, "** ALERT ERROR ** "); + break; + case LOG_CRIT: + fprintf(lctx->logfp, "** CRITICAL ERROR ** "); + break; + case LOG_ERR: + fprintf(lctx->logfp, "** ERROR ** "); + break; + case LOG_WARNING: + fprintf(lctx->logfp, "*WARNING* "); + break; + case LOG_NOTICE: + fprintf(lctx->logfp, "[NOTICE] "); + break; + case LOG_INFO: + fprintf(lctx->logfp, "[INFO] "); + break; + case LOG_DEBUG: + fprintf(lctx->logfp, "[DEBUG] "); + break; + } + vfprintf(lctx->logfp, fmt, ap); + fprintf(lctx->logfp, "\n"); + pthread_mutex_unlock(lctx->mtx_log); + + if( lctx->logtype == ltFILE ) { + fflush(lctx->logfp); + } + break; + } + va_end(ap); + } +} diff --git a/server/parser/log.h b/server/parser/log.h new file mode 100644 index 0000000..1dda0a6 --- /dev/null +++ b/server/parser/log.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file log.h + * @author David Sommerseth <davids@redhat.com> + * @date Wed Oct 21 11:38:51 2009 + * + * @brief Generic log functions + * + */ + +#ifndef _RTEVAL_LOG_H +#define _RTEVAL_LOG_H + +#include <pthread.h> +#include <syslog.h> + +/** + * Supported log types + */ +typedef enum { ltSYSLOG, ltFILE, ltCONSOLE } LogType; + +/** + * The log context structure. Keeps needed information for + * a flawless logging experience :-P + */ +typedef struct { + LogType logtype; /**< What kind of log "device" will be used */ + FILE *logfp; /**< Only used if logging to stderr, stdout or a file */ + unsigned int verbosity; /**< Defines which log level the user wants to log */ + pthread_mutex_t *mtx_log; /**< Mutex to threads to write to a file based log in parallel */ +} LogContext; + + +LogContext *init_log(const char *fname, const char *loglvl); +void close_log(LogContext *lctx); +void writelog(LogContext *lctx, unsigned int loglvl, const char *fmt, ... ); + +#endif diff --git a/server/parser/parsethread.c b/server/parser/parsethread.c new file mode 100644 index 0000000..fbf6777 --- /dev/null +++ b/server/parser/parsethread.c @@ -0,0 +1,337 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file parsethread.c + * @author David Sommerseth <davids@redhat.com> + * @date Thu Oct 15 11:52:10 2009 + * + * @brief Contains the "main" function which a parser threads runs + * + * + */ + +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <signal.h> +#include <pthread.h> +#include <libgen.h> +#include <errno.h> +#include <assert.h> + +#include <eurephia_nullsafe.h> +#include <parsethread.h> +#include <pgsql.h> +#include <log.h> +#include <threadinfo.h> +#include <statuses.h> + + +/** + * Does the same job as 'mkdir -p', but it expects a complete filename as input, and it will + * extract the directory path from that filename + * + * @param fname Full filename containing the directory the report will reside. + * + * @return Returns 1 on success, otherwise -1 + */ +static int make_report_dir(LogContext *log, const char *fname) { + char *fname_cp = NULL, *dname = NULL, *chkdir = NULL; + char *tok = NULL, *saveptr = NULL; + int ret = 0; + struct stat info; + + if( !fname ) { + return 0; + } + + fname_cp = strdup(fname); + assert( fname_cp != NULL ); + dname = dirname(fname_cp); + chkdir = malloc_nullsafe(log, strlen(dname)+8); + + if( dname[0] == '/' ) { + chkdir[0] = '/'; + } + + // Traverse the directory path, and make sure the directory exists + tok = strtok_r(dname, "/", &saveptr); + while( tok ) { + strcat(chkdir, tok); + strcat(chkdir, "/"); + + errno = 0; + // Check if directory exists + if( (stat(chkdir, &info) < 0) ) { + switch( errno ) { + case ENOENT: // If the directory do not exist, create it + if( mkdir(chkdir, 0755) < 0 ) { + // If creating dir failed, report error + writelog(log, LOG_ALERT, + "Could not create directory: %s (%s)", + chkdir, strerror(errno)); + ret = -1; + goto exit; + } + break; + default: // If other failure, report that and exit + writelog(log, LOG_ALERT, + "Could not access directory: %s (%s)", + chkdir, strerror(errno)); + ret = -1; + goto exit; + } + } + // Goto next path element + tok = strtok_r(NULL, "/", &saveptr); + } + ret = 1; + exit: + free_nullsafe(fname_cp); + free_nullsafe(chkdir); + + return ret; +} + + +/** + * Builds up a proper full path of where to save the report. + * + * @param destdir Destination directory for all reports + * @param fname Report filename, containing hostname of the reporter + * @param rterid rteval run ID + * + * @return Returns a pointer to a string with the new full path filename on success, otherwise NULL. + */ +static char *get_destination_path(LogContext *log, const char *destdir, + parseJob_t *job, const int rterid) +{ + char *newfname = NULL; + int retlen = 0; + + if( !job || rterid < 0 ) { + return NULL; + } + + retlen = strlen_nullsafe(job->clientid) + strlen(destdir) + 24; + newfname = malloc_nullsafe(log, retlen+2); + + snprintf(newfname, retlen, "%s/%s/report-%i.xml", destdir, job->clientid, rterid); + + return newfname; +} + + +/** + * The core parse function. Parses an XML file and stores it in the database according to + * the xmlparser.xsl template. + * + * @param dbc Database connection + * @param xslt Pointer to a parsed XSLT Stylesheet (xmlparser.xsl) + * @param mtx_sysreg Mutex locking to avoid simultaneous registration of systems, as they cannot + * be in an SQL transaction (due to SHA1 sysid must be registered and visible ASAP) + * @param destdir Destination directory for the report file, when moved from the queue. + * @param job Pointer to a parseJob_t structure containing the job information + * + * @return Return values: + * @code + * STAT_SUCCESS : Successfully registered report + * STAT_XMLFAIL : Could not parse the XML report file + * STAT_SYSREG : Failed to register the system into the systems or systems_hostname tables + * STAT_RTERIDREG: Failed to get a new rterid value + * STAT_GENDB : Failed to start an SQL transaction (BEGIN) + * STAT_RTEVRUNS : Failed to register the rteval run into rtevalruns or rtevalruns_details + * STAT_CYCLIC : Failed to register the data into cyclic_statistics or cyclic_rawdata tables + * STAT_REPMOVE : Failed to move the report file + * @endcode + */ +inline int parse_report(dbconn *dbc, xsltStylesheet *xslt, pthread_mutex_t *mtx_sysreg, + const char *destdir, parseJob_t *job) { + int syskey = -1, rterid = -1; + int rc = -1; + xmlDoc *repxml = NULL; + char *destfname; + + repxml = xmlParseFile(job->filename); + if( !repxml ) { + writelog(dbc->log, LOG_ERR, + "Could not parse XML file: %s", job->filename); + return STAT_XMLFAIL; + } + + pthread_mutex_lock(mtx_sysreg); + syskey = db_register_system(dbc, xslt, repxml); + if( syskey < 0 ) { + writelog(dbc->log, LOG_ERR, + "Failed to register system (XML file: %s)", job->filename); + rc = STAT_SYSREG; + goto exit; + + } + rterid = db_get_new_rterid(dbc); + if( rterid < 0 ) { + writelog(dbc->log, LOG_ERR, + "Failed to register rteval run (XML file: %s)", job->filename); + rc = STAT_RTERIDREG; + goto exit; + } + pthread_mutex_unlock(mtx_sysreg); + + if( db_begin(dbc) < 1 ) { + rc = STAT_GENDB; + goto exit; + } + + // Create a new filename of where to save the report + destfname = get_destination_path(dbc->log, destdir, job, rterid); + if( !destfname ) { + writelog(dbc->log, LOG_ERR, + "Failed to generate local report filename for (%i) %s", + job->submid, job->filename); + db_rollback(dbc); + rc = STAT_UNKNFAIL; + goto exit; + } + + if( db_register_rtevalrun(dbc, xslt, repxml, job->submid, syskey, rterid, destfname) < 0 ) { + writelog(dbc->log, LOG_ERR, + "Failed to register rteval run (XML file: %s)", + job->filename); + db_rollback(dbc); + rc = STAT_RTEVRUNS; + goto exit; + } + + if( db_register_cyclictest(dbc, xslt, repxml, rterid) != 1 ) { + writelog(dbc->log, LOG_ERR, + "Failed to register cyclictest data (XML file: %s)", + job->filename); + db_rollback(dbc); + rc = STAT_CYCLIC; + goto exit; + } + + // When all database registrations are done, move the file to it's right place + if( make_report_dir(dbc->log, destfname) < 1 ) { // Make sure report directory exists + db_rollback(dbc); + rc = STAT_REPMOVE; + goto exit; + } + + if( rename(job->filename, destfname) < 0 ) { // Move the file + writelog(dbc->log, LOG_ERR, + "Failed to move report file from %s to %s (%s)", + job->filename, destfname, strerror(errno)); + db_rollback(dbc); + rc = STAT_REPMOVE; + goto exit; + } + free_nullsafe(destfname); + + rc = STAT_SUCCESS; + db_commit(dbc); + + exit: + xmlFreeDoc(repxml); + return rc; +} + + +/** + * The parser thread. This thread lives until a shutdown notification is received. It pulls + * messages on a POSIX MQ based message queue containing submission ID and full path to an XML + * report to be parsed. + * + * @param thrargs Contains database connection, XSLT stylesheet, POSXI MQ descriptor, etc + * + * @return Returns 0 on successful operation, otherwise 1 on errors. + */ +void *parsethread(void *thrargs) { + threadData_t *args = (threadData_t *) thrargs; + parseJob_t jobinfo; + long exitcode = 0; + + writelog(args->dbc->log, LOG_DEBUG, "[Thread %i] Starting", args->id); + pthread_mutex_lock(args->mtx_thrcnt); + (*(args->threadcount)) += 1; + pthread_mutex_unlock(args->mtx_thrcnt); + + // Polling loop + while( *(args->shutdown) == 0 ) { + int len = 0; + unsigned int prio = 0; + + // Check if the database connection is alive before pulling any messages + if( db_ping(args->dbc) != 1 ) { + writelog(args->dbc->log, LOG_EMERG, + "[Thread %i] Lost database conneciting: Shutting down thread.", + args->id); + + if( *(args->threadcount) <= 1 ) { + writelog(args->dbc->log, LOG_EMERG, + "No more worker threads available. " + "Initiating complete shutdown"); + kill(getpid(), SIGUSR1); + } + exitcode = 1; + goto exit; + } + + // Retrieve a parse job from the message queue + memset(&jobinfo, 0, sizeof(parseJob_t)); + errno = 0; + len = mq_receive(args->msgq, (char *)&jobinfo, sizeof(parseJob_t), &prio); + if( (len < 0) && errno != EAGAIN ) { + writelog(args->dbc->log, LOG_CRIT, + "Could not receive the message from queue: %s", + strerror(errno)); + pthread_exit((void *) 1); + } + + // Ignore whatever message if the shutdown flag is set. + if( *(args->shutdown) != 0 ) { + break; + } + + // If we have a message, then process the parse job + if( (errno != EAGAIN) && (len > 0) ) { + int res = 0; + + writelog(args->dbc->log, LOG_DEBUG, + "** Thread %i: Job recieved, submid: %i", + args->id, jobinfo.submid); + + // Mark the job as "in progress", if successful update, continue parsing it + if( db_update_submissionqueue(args->dbc, jobinfo.submid, STAT_INPROG) ) { + res = parse_report(args->dbc, args->xslt, args->mtx_sysreg, + args->destdir, &jobinfo); + // Set the status for the submission + db_update_submissionqueue(args->dbc, jobinfo.submid, res); + } else { + writelog(args->dbc->log, LOG_CRIT, + "Failed to mark submid %i as STAT_INPROG", + jobinfo.submid); + } + } + } + writelog(args->dbc->log, LOG_DEBUG, "[Thread %i] Shut down", args->id); + exit: + pthread_mutex_lock(args->mtx_thrcnt); + (*(args->threadcount)) -= 1; + pthread_mutex_unlock(args->mtx_thrcnt); + + pthread_exit((void *) exitcode); +} diff --git a/server/parser/parsethread.h b/server/parser/parsethread.h new file mode 100644 index 0000000..66e8524 --- /dev/null +++ b/server/parser/parsethread.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file parsethread.h + * @author David Sommerseth <davids@redhat.com> + * @date Thu Oct 15 11:52:10 2009 + * + * @brief Contains the "main" function which a parser threads runs + * + */ + +#ifndef _PARSETHREAD_H +#define _PARSETHREAD_H + +/** + * jbNONE means no job available, + * jbAVAIL indicates that parseJob_t contains a job +*/ +typedef enum { jbNONE, jbAVAIL } jobStatus; + +/** + * This struct is used for sending a parse job to a worker thread via POSIX MQ + */ +typedef struct { + jobStatus status; /**< Info about if job information*/ + unsigned int submid; /**< Work info: Numeric ID of the job being parsed */ + char clientid[256]; /**< Work info: Should contain senders hostname */ + char filename[4096]; /**< Work info: Full filename of the report to be parsed */ +} parseJob_t; + + +void *parsethread(void *thrargs); + +#endif diff --git a/server/parser/pgsql.c b/server/parser/pgsql.c new file mode 100644 index 0000000..7a6e7ec --- /dev/null +++ b/server/parser/pgsql.c @@ -0,0 +1,989 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file pgsql.c + * @author David Sommerseth <davids@redhat.com> + * @date Wed Oct 13 17:44:35 2009 + * + * @brief Database API for the PostgreSQL database. + * + * + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pthread.h> +#include <assert.h> +#include <errno.h> + +#include <libpq-fe.h> + +#include <libxml/parser.h> +#include <libxml/xmlsave.h> +#include <libxslt/transform.h> +#include <libxslt/xsltutils.h> + +#include <eurephia_nullsafe.h> +#include <eurephia_xml.h> +#include <eurephia_values.h> +#include <configparser.h> +#include <xmlparser.h> +#include <pgsql.h> +#include <log.h> +#include <statuses.h> + +/** + * Connect to a database, based on the given configuration + * + * @param cfg eurephiaVALUES containing the configuration + * @param id Database connection ID. Used to identify which thread is doing what with the database + * @param log Log context, where all logging will go + * + * @return Returns a database connection context + */ +dbconn *db_connect(eurephiaVALUES *cfg, unsigned int id, LogContext *log) { + dbconn *ret = NULL; + + ret = (dbconn *) malloc_nullsafe(log, sizeof(dbconn)+2); + ret->id = id; + ret->log = log; + + writelog(log, LOG_DEBUG, "[Connection %i] Connecting to database: server=%s:%s, " + "database=%s, user=%s", ret->id, + eGet_value(cfg, "db_server"), eGet_value(cfg, "db_port"), + eGet_value(cfg, "database"), eGet_value(cfg, "db_username")); + ret->db = PQsetdbLogin(eGet_value(cfg, "db_server"), + eGet_value(cfg, "db_port"), + NULL, /* pgopt */ + NULL, /* pgtty */ + eGet_value(cfg, "database"), + eGet_value(cfg, "db_username"), + eGet_value(cfg, "db_password")); + + if( !ret->db ) { + writelog(log, LOG_EMERG, + "[Connection %i] Could not connect to the database (unknown reason)", ret->id); + free_nullsafe(ret); + return NULL; + } + + if( PQstatus(ret->db) != CONNECTION_OK ) { + writelog(log, LOG_EMERG, "[Connection %i] Failed to connect to the database: %s", + ret->id, PQerrorMessage(ret->db)); + free_nullsafe(ret); + return NULL; + } + return ret; +} + + +/** + * Pings the database connection to check if it is alive + * + * @param dbc Database connection to ping + * + * @return Returns 1 if the connection is alive, otherwise 0 + */ +int db_ping(dbconn *dbc) { + PGresult *res = NULL; + + // Send ping + res = PQexec(dbc->db, ""); + PQclear(res); + + // Check status + if( PQstatus(dbc->db) != CONNECTION_OK ) { + PQreset(dbc->db); + if( PQstatus(dbc->db) != CONNECTION_OK ) { + writelog(dbc->log, LOG_EMERG, + "[Connection %i] Database error - Lost connection: %s", + dbc->id, PQerrorMessage(dbc->db)); + return 0; + } else { + writelog(dbc->log, LOG_CRIT, + "[Conncetion %i] Database connection restored", dbc->id); + } + } + return 1; +} + + +/** + * Disconnect from the database + * + * @param dbc Pointer to the database handle to be disconnected. + */ +void db_disconnect(dbconn *dbc) { + if( dbc && dbc->db ) { + writelog(dbc->log, LOG_DEBUG, "[Connection %i] Disconnecting from database", dbc->id); + PQfinish(dbc->db); + dbc->db = NULL; + dbc->log = NULL; + } + free_nullsafe(dbc); +} + + +/** + * This function does INSERT SQL queries based on an XML document (sqldata) which contains + * all information about table, fields and records to be inserted. For security and performance, + * this function uses prepared SQL statements. + * + * This function is PostgreSQL specific. + * + * @param dbc Database handler to a PostgreSQL + * @param sqldoc sqldata XML document containing the data to be inserted. + * + * The sqldata XML document must be formated like this: + * @code + * <sqldata table="{table name}" [key="{field name}"> + * <fields> + * <field fid="{integer}">{field name}</field> + * ... + * ... + * <field fid="{integer_n}">{field name 'n'}</field> + * </fields> + * <records> + * <record> + * <value fid="{integer} [type="{data type}"] [hash="{hash type}">{value for field 'fid'</value> + * ... + * ... + * <value fid="{integer_n}">{value for field 'fid_n'</value> + * </record> + * ... + * ... + * ... + * </records> + * </sqldata> + * @endcode + * The 'sqldata' root tag must contain a 'table' attribute. This must contain the a name of a table + * in the database. If the 'key' attribute is set, the function will return the that field value for + * each INSERT query, using INSERT ... RETURNING {field name}. The sqldata root tag must then have + * two children, 'fields' and 'records'. + * + * The 'fields' tag need to contain 'field' children tags for each field to insert data for. Each + * field in the fields tag must be assigned a unique integer. + * + * The 'records' tag need to contain 'record' children tags for each record to be inserted. Each + * record tag needs to have 'value' tags for each field which is found in the 'fields' section. + * + * The 'value' tags must have a 'fid' attribute. This is the link between the field name in the + * 'fields' section and the value to be inserted. + * + * The 'type' attribute may be used as well, but the only supported data type supported to this + * attribute is 'xmlblob'. In this case, the contents of the 'value' tag must be more XML tags. + * These tags will then be serialised to a string which is inserted into the database. + * + * The 'hash' attribute of the 'value' tag can be set to 'sha1'. This will make do a SHA1 hash + * calculation of the value and this hash value will be used for the insert. + * + * @return Returns an eurephiaVALUES list containing information about each record which was inserted. + * If the 'key' attribute is not set in the 'sqldata' tag, the OID value of each record will be + * saved. If the table do not support OIDs, the value will be '0'. Otherwise the contents of + * the defined field name will be returned. If one of the INSERT queries fails, it will abort + * further processing and the function will return NULL. + */ +eurephiaVALUES *pgsql_INSERT(dbconn *dbc, xmlDoc *sqldoc) { + xmlNode *root_n = NULL, *fields_n = NULL, *recs_n = NULL, *ptr_n = NULL, *val_n = NULL; + char **field_ar = NULL, *fields = NULL, **value_ar = NULL, *values = NULL, *table = NULL, + tmp[20], *sql = NULL, *key = NULL; + unsigned int fieldcnt = 0, *field_idx, i = 0; + PGresult *dbres = NULL; + eurephiaVALUES *res = NULL; + + assert( (dbc != NULL) && (sqldoc != NULL) ); + + root_n = xmlDocGetRootElement(sqldoc); + if( !root_n || (xmlStrcmp(root_n->name, (xmlChar *) "sqldata") != 0) ) { + writelog(dbc->log, LOG_ERR, + "[Connection %i] Input XML document is not a valid sqldata document", dbc->id); + return NULL; + } + + table = xmlGetAttrValue(root_n->properties, "table"); + if( !table ) { + writelog(dbc->log, LOG_ERR, + "[Connection %i] Input XML document is missing table reference", dbc->id); + return NULL; + } + + key = xmlGetAttrValue(root_n->properties, "key"); + + fields_n = xmlFindNode(root_n, "fields"); + recs_n = xmlFindNode(root_n, "records"); + if( !fields_n || !recs_n ) { + writelog(dbc->log, LOG_ERR, + "[Connection %i] Input XML document is missing either <fields/> or <records/>", + dbc->id); + return NULL; + } + + // Count number of fields + foreach_xmlnode(fields_n->children, ptr_n) { + if( ptr_n->type == XML_ELEMENT_NODE ) { + fieldcnt++; + } + } + + // Generate lists of all fields and a index mapping table + field_idx = calloc(fieldcnt+1, sizeof(unsigned int)); + field_ar = calloc(fieldcnt+1, sizeof(char *)); + foreach_xmlnode(fields_n->children, ptr_n) { + if( ptr_n->type != XML_ELEMENT_NODE ) { + continue; + } + + field_idx[i] = atoi_nullsafe(xmlGetAttrValue(ptr_n->properties, "fid")); + field_ar[i] = xmlExtractContent(ptr_n); + i++; + } + + // Generate strings with field names and value place holders + // for a prepared SQL statement + fields = malloc_nullsafe(dbc->log, 3); + values = malloc_nullsafe(dbc->log, 6*(fieldcnt+1)); + strcpy(fields, "("); + strcpy(values, "("); + int len = 3; + for( i = 0; i < fieldcnt; i++ ) { + // Prepare VALUES section + snprintf(tmp, 6, "$%i", i+1); + append_str(values, tmp, (6*fieldcnt)); + + // Prepare fields section + len += strlen_nullsafe(field_ar[i])+2; + fields = realloc(fields, len); + strcat(fields, field_ar[i]); + + if( i < (fieldcnt-1) ) { + strcat(fields, ","); + strcat(values, ","); + } + } + strcat(fields, ")"); + strcat(values, ")"); + + // Build up the SQL query + sql = malloc_nullsafe(dbc->log, + strlen_nullsafe(fields) + + strlen_nullsafe(values) + + strlen_nullsafe(table) + + strlen_nullsafe(key) + + 34 /* INSERT INTO VALUES RETURNING*/ + ); + sprintf(sql, "INSERT INTO %s %s VALUES %s", table, fields, values); + if( key ) { + strcat(sql, " RETURNING "); + strcat(sql, key); + } + + // Create a prepared SQL query +#ifdef DEBUG_SQL + writelog(dbc->log, LOG_DEBUG, "[Connection %i] Preparing SQL statement: %s", dbc->id, sql); +#endif + dbres = PQprepare(dbc->db, "", sql, fieldcnt, NULL); + if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) { + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Failed to prepare SQL query: %s", + dbc->id, PQresultErrorMessage(dbres)); + PQclear(dbres); + goto exit; + } + PQclear(dbres); + + // Loop through all records and generate SQL statements + res = eCreate_value_space(dbc->log, 1); + foreach_xmlnode(recs_n->children, ptr_n) { + if( ptr_n->type != XML_ELEMENT_NODE ) { + continue; + } + + // Loop through all value nodes in each record node and get the values for each field + value_ar = calloc(fieldcnt, sizeof(char *)); + i = 0; + foreach_xmlnode(ptr_n->children, val_n) { + char *fid_s = NULL; + int fid = -1; + + if( i > fieldcnt ) { + break; + } + + if( val_n->type != XML_ELEMENT_NODE ) { + continue; + } + + fid_s = xmlGetAttrValue(val_n->properties, "fid"); + fid = atoi_nullsafe(fid_s); + if( (fid_s == NULL) || (fid < 0) ) { + continue; + } + value_ar[field_idx[i]] = sqldataExtractContent(dbc->log, val_n); + i++; + } + + // Insert the record into the database + dbres = PQexecPrepared(dbc->db, "", fieldcnt, + (const char * const *)value_ar, NULL, NULL, 0); + if( PQresultStatus(dbres) != (key ? PGRES_TUPLES_OK : PGRES_COMMAND_OK) ) { + writelog(dbc->log, LOG_ALERT, "[Connection %i] Failed to do SQL INSERT query: %s", + dbc->id, PQresultErrorMessage(dbres)); + PQclear(dbres); + eFree_values(res); + res = NULL; + + // Free up the memory we've used for this record + for( i = 0; i < fieldcnt; i++ ) { + free_nullsafe(value_ar[i]); + } + free_nullsafe(value_ar); + goto exit; + } + if( key ) { + // If the /sqldata/@key attribute was set, fetch the returning ID + eAdd_value(res, key, PQgetvalue(dbres, 0, 0)); + } else { + static char oid[32]; + snprintf(oid, 30, "%ld%c", (unsigned long int) PQoidValue(dbres), 0); + eAdd_value(res, "oid", oid); + } + PQclear(dbres); + + // Free up the memory we've used for this record + for( i = 0; i < fieldcnt; i++ ) { + free_nullsafe(value_ar[i]); + } + free_nullsafe(value_ar); + } + + exit: + free_nullsafe(sql); + free_nullsafe(fields); + free_nullsafe(values); + free_nullsafe(field_ar); + free_nullsafe(field_idx); + return res; +} + + +/** + * Start an SQL transaction (SQL BEGIN) + * + * @param dbc Database handler where to perform the SQL queries + * + * @return Returns 1 on success, otherwise -1 is returned + */ +int db_begin(dbconn *dbc) { + PGresult *dbres = NULL; + + dbres = PQexec(dbc->db, "BEGIN"); + if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) { + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Failed to do prepare a transaction (BEGIN): %s", + dbc->id, PQresultErrorMessage(dbres)); + PQclear(dbres); + return -1; + } + PQclear(dbres); + return 1; +} + + +/** + * Commits an SQL transaction (SQL COMMIT) + * + * @param dbc Database handler where to perform the SQL queries + * + * @return Returns 1 on success, otherwise -1 is returned + */ +int db_commit(dbconn *dbc) { + PGresult *dbres = NULL; + + dbres = PQexec(dbc->db, "COMMIT"); + if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) { + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Failed to do commit a database transaction (COMMIT): %s", + dbc->id, PQresultErrorMessage(dbres)); + PQclear(dbres); + return -1; + } + PQclear(dbres); + return 1; +} + + +/** + * Aborts an SQL transaction (SQL ROLLBACK/ABORT) + * + * @param dbc Database handler where to perform the SQL queries + * + * @return Returns 1 on success, otherwise -1 is returned + */ +int db_rollback(dbconn *dbc) { + PGresult *dbres = NULL; + + dbres = PQexec(dbc->db, "ROLLBACK"); + if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) { + writelog(dbc->log, LOG_CRIT, + "[Connection %i] Failed to do abort/rollback a transaction (ROLLBACK): %s", + dbc->id, PQresultErrorMessage(dbres)); + PQclear(dbres); + return -1; + } + PQclear(dbres); + return 1; +} + + +/** + * This function blocks until a notification is received from the database + * + * @param dbc Database connection + * @param shutdown Pointer to the shutdown flag. Used to avoid reporting false errors. + * @param listenfor Name to be used when calling LISTEN + * + * @return Returns 1 on successful waiting, otherwise -1 + */ +int db_wait_notification(dbconn *dbc, const int *shutdown, const char *listenfor) { + int sock, ret = 0; + PGresult *dbres = NULL; + PGnotify *notify = NULL; + fd_set input_mask; + char *sql = NULL; + + sql = malloc_nullsafe(dbc->log, strlen_nullsafe(listenfor) + 12); + assert( sql != NULL ); + + // Initiate listening + sprintf(sql, "LISTEN %s", listenfor); + dbres = PQexec(dbc->db, sql); + if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) { + writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s", + dbc->id, PQresultErrorMessage(dbres)); + free_nullsafe(sql); + PQclear(dbres); + return -1; + } + PQclear(dbres); + + // Start listening and waiting + while( ret == 0 ) { + sock = PQsocket(dbc->db); + if (sock < 0) { + // shouldn't happen + ret = -1; + break; + } + + // Wait for something to happen on the database socket + FD_ZERO(&input_mask); + FD_SET(sock, &input_mask); + if (select(sock + 1, &input_mask, NULL, NULL, NULL) < 0) { + // If the shutdown flag is set, select() will fail due to a signal. Only + // report errors if we're not shutting down, or else exit normally with + // successful waiting. + if( *shutdown == 0 ) { + writelog(dbc->log, LOG_CRIT, "[Connection %i] select() failed: %s", + dbc->id, strerror(errno)); + ret = -1; + goto exit; + } else { + ret = 1; + } + break; + } + + // Process the event + PQconsumeInput(dbc->db); + + // Check if connection still is valid + if( PQstatus(dbc->db) != CONNECTION_OK ) { + PQreset(dbc->db); + if( PQstatus(dbc->db) != CONNECTION_OK ) { + writelog(dbc->log, LOG_EMERG, + "[Connection %i] Database connection died: %s", + dbc->id, PQerrorMessage(dbc->db)); + ret = -1; + goto exit; + } + writelog(dbc->log, LOG_CRIT, + "[Connection %i] Database connection restored", dbc->id); + } + + while ((notify = PQnotifies(dbc->db)) != NULL) { + // If a notification was received, inform and exit with success. + writelog(dbc->log, LOG_DEBUG, + "[Connection %i] Received notfication from pid %d", + dbc->id, notify->be_pid); + PQfreemem(notify); + ret = 1; + break; + } + } + + // Stop listening when we exit + sprintf(sql, "UNLISTEN %s", listenfor); + dbres = PQexec(dbc->db, sql); + if( PQresultStatus(dbres) != PGRES_COMMAND_OK ) { + writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s", + dbc->id, PQresultErrorMessage(dbres)); + free_nullsafe(sql); + ret = -1; + } + free_nullsafe(sql); + PQclear(dbres); + + exit: + return ret; +} + + +/** + * Retrive the first available submitted report + * + * @param dbc Database connection + * @param mtx pthread_mutex to avoid parallel access to the submission queue table, to avoid + * the same job being retrieved multiple times. + * + * @return Returns a pointer to a parseJob_t struct, with the parse job info on success, otherwise NULL + */ +parseJob_t *db_get_submissionqueue_job(dbconn *dbc, pthread_mutex_t *mtx) { + parseJob_t *job = NULL; + PGresult *res = NULL; + char sql[4098]; + + job = (parseJob_t *) malloc_nullsafe(dbc->log, sizeof(parseJob_t)); + + // Get the first available submission + memset(&sql, 0, 4098); + snprintf(sql, 4096, + "SELECT submid, filename, clientid" + " FROM submissionqueue" + " WHERE status = %i" + " ORDER BY submid" + " LIMIT 1", + STAT_NEW); + + pthread_mutex_lock(mtx); + res = PQexec(dbc->db, sql); + if( PQresultStatus(res) != PGRES_TUPLES_OK ) { + pthread_mutex_unlock(mtx); + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Failed to query submission queue (SELECT): %s", + dbc->id, PQresultErrorMessage(res)); + PQclear(res); + free_nullsafe(job); + return NULL; + } + + if( PQntuples(res) == 1 ) { + job->status = jbAVAIL; + job->submid = atoi_nullsafe(PQgetvalue(res, 0, 0)); + snprintf(job->filename, 4095, "%.4094s", PQgetvalue(res, 0, 1)); + snprintf(job->clientid, 255, "%.254s", PQgetvalue(res, 0, 2)); + + // Update the submission queue status + if( db_update_submissionqueue(dbc, job->submid, STAT_ASSIGNED) < 1 ) { + pthread_mutex_unlock(mtx); + writelog(dbc->log, LOG_ALERT, "[Connection %i] Failed to update " + "submission queue statis to STAT_ASSIGNED", dbc->id); + free_nullsafe(job); + return NULL; + } + } else { + job->status = jbNONE; + } + pthread_mutex_unlock(mtx); + PQclear(res); + return job; +} + + +/** + * Updates the submission queue table with the new status and the appropriate timestamps + * + * @param dbc Database handler to the rteval database + * @param submid Submission ID to update + * @param status The new status + * + * @return Returns 1 on success, 0 on invalid status ID and -1 on database errors. + */ +int db_update_submissionqueue(dbconn *dbc, unsigned int submid, int status) { + PGresult *res = NULL; + char sql[4098]; + + memset(&sql, 0, 4098); + switch( status ) { + case STAT_ASSIGNED: + case STAT_RTERIDREG: + case STAT_REPMOVE: + snprintf(sql, 4096, + "UPDATE submissionqueue SET status = %i" + " WHERE submid = %i", status, submid); + break; + + case STAT_INPROG: + snprintf(sql, 4096, + "UPDATE submissionqueue SET status = %i, parsestart = NOW()" + " WHERE submid = %i", status, submid); + break; + + case STAT_SUCCESS: + case STAT_UNKNFAIL: + case STAT_XMLFAIL: + case STAT_SYSREG: + case STAT_GENDB: + case STAT_RTEVRUNS: + case STAT_CYCLIC: + snprintf(sql, 4096, + "UPDATE submissionqueue SET status = %i, parseend = NOW() WHERE submid = %i", + status, submid); + break; + + default: + case STAT_NEW: + writelog(dbc->log, LOG_ERR, + "[Connection %i] Invalid status (%i) attempted to set on submid %i", + dbc->id, status, submid); + return 0; + } + + res = PQexec(dbc->db, sql); + if( !res ) { + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Unkown error when updating submid %i to status %i", + dbc->id, submid, status); + return -1; + } else if( PQresultStatus(res) != PGRES_COMMAND_OK ) { + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Failed to UPDATE submissionqueue (submid: %i, status: %i): %s", + dbc->id, submid, status, PQresultErrorMessage(res)); + PQclear(res); + return -1; + } + PQclear(res); + return 1; +} + + +/** + * Registers information into the 'systems' and 'systems_hostname' tables, based on the + * summary/report XML file from rteval. + * + * @param dbc Database handler where to perform the SQL queries + * @param xslt A pointer to a parsed 'xmlparser.xsl' XSLT template + * @param summaryxml The XML report from rteval + * + * @return Returns a value > 0 on success, which is a unique reference to the system of the report. + * If the function detects that this system is already registered, the 'syskey' reference will + * be reused. On errors, -1 will be returned. + */ +int db_register_system(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml) { + PGresult *dbres = NULL; + eurephiaVALUES *dbdata = NULL; + xmlDoc *sysinfo_d = NULL, *hostinfo_d = NULL; + parseParams prms; + char sqlq[4098]; + char *sysid = NULL; // SHA1 value of the system id + char *ipaddr = NULL, *hostname = NULL; + int syskey = -1; + + memset(&prms, 0, sizeof(parseParams)); + prms.table = "systems"; + sysinfo_d = parseToSQLdata(dbc->log, xslt, summaryxml, &prms); + if( !sysinfo_d ) { + writelog(dbc->log, LOG_ERR, "[Connection %i] Could not parse the input XML data", dbc->id); + syskey= -1; + goto exit; + } + sysid = sqldataGetValue(dbc->log, sysinfo_d, "sysid", 0); + if( !sysid ) { + writelog(dbc->log, LOG_ERR, + "[Connection %i] Could not retrieve the sysid field from the input XML", dbc->id); + syskey= -1; + goto exit; + } + + memset(&sqlq, 0, 4098); + snprintf(sqlq, 4096, "SELECT syskey FROM systems WHERE sysid = '%.256s'", sysid); + free_nullsafe(sysid); + dbres = PQexec(dbc->db, sqlq); + if( PQresultStatus(dbres) != PGRES_TUPLES_OK ) { + writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s", + dbc->id, PQresultErrorMessage(dbres)); + writelog(dbc->log, LOG_DEBUG, "[Connection %i] Failing SQL query: %s", + dbc->id, sqlq); + PQclear(dbres); + syskey= -1; + goto exit; + } + + if( PQntuples(dbres) == 0 ) { // No record found, need to register this system + PQclear(dbres); + + dbdata = pgsql_INSERT(dbc, sysinfo_d); + if( !dbdata ) { + syskey= -1; + goto exit; + } + if( (eCount(dbdata) != 1) || !dbdata->val ) { // Only one record should be registered + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Failed to register the system", dbc->id); + eFree_values(dbdata); + syskey= -1; + goto exit; + } + syskey = atoi_nullsafe(dbdata->val); + hostinfo_d = sqldataGetHostInfo(dbc->log, xslt, summaryxml, syskey, &hostname, &ipaddr); + if( !hostinfo_d ) { + syskey = -1; + goto exit; + } + eFree_values(dbdata); + + dbdata = pgsql_INSERT(dbc, hostinfo_d); + syskey = (dbdata ? syskey : -1); + eFree_values(dbdata); + + } else if( PQntuples(dbres) == 1 ) { // System found - check if the host IP is known or not + syskey = atoi_nullsafe(PQgetvalue(dbres, 0, 0)); + hostinfo_d = sqldataGetHostInfo(dbc->log, xslt, summaryxml, syskey, &hostname, &ipaddr); + if( !hostinfo_d ) { + syskey = -1; + goto exit; + } + PQclear(dbres); + + // Check if this hostname and IP address is registered + snprintf(sqlq, 4096, + "SELECT syskey FROM systems_hostname" + " WHERE hostname='%.256s' AND ipaddr='%.64s'", + hostname, ipaddr); + dbres = PQexec(dbc->db, sqlq); + if( PQresultStatus(dbres) != PGRES_TUPLES_OK ) { + writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s", + dbc->id, PQresultErrorMessage(dbres)); + writelog(dbc->log, LOG_DEBUG, "[Connection %i] Failing SQL query: %s", + dbc->id, sqlq); + PQclear(dbres); + syskey= -1; + goto exit; + } + + if( PQntuples(dbres) == 0 ) { // Not registered, then register it + dbdata = pgsql_INSERT(dbc, hostinfo_d); + syskey = (dbdata ? syskey : -1); + eFree_values(dbdata); + } + PQclear(dbres); + } else { + // Critical -- system IDs should not be registered more than once + writelog(dbc->log, LOG_CRIT, "[Connection %i] Multiple systems registered (%s)", + dbc->id, sqlq); + syskey= -1; + } + + exit: + free_nullsafe(hostname); + free_nullsafe(ipaddr); + if( sysinfo_d ) { + xmlFreeDoc(sysinfo_d); + } + if( hostinfo_d ) { + xmlFreeDoc(hostinfo_d); + } + return syskey; +} + + +/** + * Retrieves the next available rteval run ID (rterid) + * + * @param dbc Database handler where to perform the SQL query + * + * @return Returns a value > 0 on success, containing the assigned rterid value. Otherwise -1 is returned. + */ +int db_get_new_rterid(dbconn *dbc) { + PGresult *dbres = NULL; + int rterid = 0; + + dbres = PQexec(dbc->db, "SELECT nextval('rtevalruns_rterid_seq')"); + if( (PQresultStatus(dbres) != PGRES_TUPLES_OK) || (PQntuples(dbres) != 1) ) { + rterid = -1; + } else { + rterid = atoi_nullsafe(PQgetvalue(dbres, 0, 0)); + } + + if( rterid < 1 ) { + writelog(dbc->log, LOG_CRIT, + "[Connection %i] Failed to retrieve a new rterid value", dbc->id); + } + if( rterid < 0 ) { + writelog(dbc->log, LOG_ALERT, "[Connection %i] SQL %s", + dbc->id, PQresultErrorMessage(dbres)); + } + PQclear(dbres); + return rterid; +} + + +/** + * Registers information into the 'rtevalruns' and 'rtevalruns_details' tables + * + * @param dbc Database handler where to perform the SQL queries + * @param xslt A pointer to a parsed 'xmlparser.xsl' XSLT template + * @param summaryxml The XML report from rteval + * @param submid Submission ID, referencing the record in the submissionqueue table. + * @param syskey A positive integer containing the return value from db_register_system() + * @param rterid A positive integer containing the return value from db_get_new_rterid() + * @param report_fname A string containing the filename of the report. + * + * @return Returns 1 on success, otherwise -1 is returned. + */ +int db_register_rtevalrun(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml, + unsigned int submid, int syskey, int rterid, const char *report_fname) +{ + int ret = -1; + xmlDoc *rtevalrun_d = NULL, *rtevalrundets_d = NULL; + parseParams prms; + eurephiaVALUES *dbdata = NULL; + + // Parse the rtevalruns information + memset(&prms, 0, sizeof(parseParams)); + prms.table = "rtevalruns"; + prms.syskey = syskey; + prms.rterid = rterid; + prms.submid = submid; + prms.report_filename = report_fname; + rtevalrun_d = parseToSQLdata(dbc->log, xslt, summaryxml, &prms); + if( !rtevalrun_d ) { + writelog(dbc->log, LOG_ERR, + "[Connection %i] Could not parse the input XML data", dbc->id); + ret = -1; + goto exit; + } + + // Register the rteval run information + dbdata = pgsql_INSERT(dbc, rtevalrun_d); + if( !dbdata ) { + ret = -1; + goto exit; + } + + if( eCount(dbdata) != 1 ) { + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Failed to register the rteval run", dbc->id); + ret = -1; + eFree_values(dbdata); + goto exit; + } + eFree_values(dbdata); + + // Parse the rtevalruns_details information + memset(&prms, 0, sizeof(parseParams)); + prms.table = "rtevalruns_details"; + prms.rterid = rterid; + rtevalrundets_d = parseToSQLdata(dbc->log, xslt, summaryxml, &prms); + if( !rtevalrundets_d ) { + writelog(dbc->log, LOG_ERR, + "[Connection %i] Could not parse the input XML data (rtevalruns_details)", + dbc->id); + ret = -1; + goto exit; + } + + // Register the rteval_details information + dbdata = pgsql_INSERT(dbc, rtevalrundets_d); + if( !dbdata ) { + ret = -1; + goto exit; + } + + // Check that only one record was inserted + if( eCount(dbdata) != 1 ) { + writelog(dbc->log, LOG_ALERT, + "[Connection %i] Failed to register the rteval run details", dbc->id); + ret = -1; + } + eFree_values(dbdata); + ret = 1; + exit: + if( rtevalrun_d ) { + xmlFreeDoc(rtevalrun_d); + } + if( rtevalrundets_d ) { + xmlFreeDoc(rtevalrundets_d); + } + return ret; +} + + +/** + * Registers data returned from cyclictest into the database. + * + * @param dbc Database handler where to perform the SQL queries + * @param xslt A pointer to a parsed 'xmlparser.xsl' XSLT template + * @param summaryxml The XML report from rteval + * @param rterid A positive integer referencing the rteval run ID, returned from db_register_rtevalrun() + * + * @return Returns 1 on success, otherwise -1 + */ +int db_register_cyclictest(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml, int rterid) { + int result = -1; + xmlDoc *cyclic_d = NULL; + parseParams prms; + eurephiaVALUES *dbdata = NULL; + int cyclicdata = 0; + const char *cyclictables[] = { "cyclic_statistics", "cyclic_histogram", "cyclic_rawdata", NULL }; + int i; + + memset(&prms, 0, sizeof(parseParams)); + prms.rterid = rterid; + + // Register the cyclictest data + for( i = 0; cyclictables[i]; i++ ) { + prms.table = cyclictables[i]; + cyclic_d = parseToSQLdata(dbc->log, xslt, summaryxml, &prms); + if( cyclic_d && cyclic_d->children ) { + // Insert SQL data which was found and generated + dbdata = pgsql_INSERT(dbc, cyclic_d); + if( !dbdata ) { + result = -1; + xmlFreeDoc(cyclic_d); + goto exit; + } + + if (eCount(dbdata) > 0) { + cyclicdata++; + } + eFree_values(dbdata); + cyclicdata = 1; + } + if( cyclic_d ) { + xmlFreeDoc(cyclic_d); + } + } + + // Report error if not enough cyclictest data is registered. + if( cyclicdata > 1 ) { + writelog(dbc->log, LOG_ALERT, + "[Connection %i] No cyclictest raw data or histogram data registered", dbc->id); + result = -1; + } else { + result = 1; + } + exit: + return result; +} diff --git a/server/parser/pgsql.h b/server/parser/pgsql.h new file mode 100644 index 0000000..e40b7d2 --- /dev/null +++ b/server/parser/pgsql.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file pgsql.h + * @author David Sommerseth <davids@redhat.com> + * @date Wed Oct 13 17:44:35 2009 + * + * @brief Database API for the PostgreSQL database. + * + * + */ + +#ifndef _RTEVAL_PGSQL_H +#define _RTEVAL_PGSQL_H + +#include <libpq-fe.h> +#include <libxml/parser.h> +#include <libxslt/transform.h> + +#include <log.h> +#include <eurephia_values.h> +#include <parsethread.h> + +/** + * A unified database abstraction layer, providing log support + */ +typedef struct { + unsigned int id; /**< Unique connection ID, used for debugging */ + LogContext *log; /**< Initialised log context */ + PGconn *db; /**< Database connection handler */ +} dbconn; + +/* Generic database function */ +dbconn *db_connect(eurephiaVALUES *cfg, unsigned int id, LogContext *log); +int db_ping(dbconn *dbc); +void db_disconnect(dbconn *dbc); +int db_begin(dbconn *dbc); +int db_commit(dbconn *dbc); +int db_rollback(dbconn *dbc); + +/* rteval specific database functions */ +int db_wait_notification(dbconn *dbc, const int *shutdown, const char *listenfor); +parseJob_t *db_get_submissionqueue_job(dbconn *dbc, pthread_mutex_t *mtx); +int db_update_submissionqueue(dbconn *dbc, unsigned int submid, int status); +int db_register_system(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml); +int db_get_new_rterid(dbconn *dbc); +int db_register_rtevalrun(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml, + unsigned int submid, int syskey, int rterid, const char *report_fname); +int db_register_cyclictest(dbconn *dbc, xsltStylesheet *xslt, xmlDoc *summaryxml, int rterid); + +#endif diff --git a/server/parser/rteval_parserd.c b/server/parser/rteval_parserd.c new file mode 100644 index 0000000..a075ff0 --- /dev/null +++ b/server/parser/rteval_parserd.c @@ -0,0 +1,521 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file rteval_parserd.c + * @author David Sommerseth <davids@redhat.com> + * @date Thu Oct 15 11:59:27 2009 + * + * @brief Polls the rteval.submissionqueue table for notifications + * from new inserts and sends the file to a processing thread + * + * + * + */ + +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <signal.h> +#include <pthread.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> +#include <assert.h> + +#include <eurephia_nullsafe.h> +#include <eurephia_values.h> +#include <configparser.h> +#include <pgsql.h> +#include <threadinfo.h> +#include <parsethread.h> +#include <argparser.h> + +#define DEFAULT_MSG_MAX 5 /**< Default size of the message queue */ +#define XMLPARSER_XSL "xmlparser.xsl" /**< rteval report parser XSLT, parses XML into database friendly data*/ + +static int shutdown = 0; /**< Variable indicating if the program should shutdown */ +static LogContext *logctx = NULL; /**< Initialsed log context, to be used by sigcatch() */ + + +/** + * Simple signal catcher. Used for SIGINT and SIGTERM signals, and will set the global shutdown + * shutdown flag. It's expected that all threads behaves properly and exits as soon as their current + * work is completed + * + * @param sig Recieved signal (not used) + */ +void sigcatch(int sig) { + switch( sig ) { + case SIGINT: + case SIGTERM: + if( shutdown == 0 ) { + shutdown = 1; + writelog(logctx, LOG_INFO, "[SIGNAL] Shutting down"); + } else { + writelog(logctx, LOG_INFO, "[SIGNAL] Shutdown in progress ... please be patient ..."); + } + break; + + case SIGUSR1: + writelog(logctx, LOG_EMERG, "[SIGNAL] Shutdown alarm from a worker thread"); + shutdown = 1; + break; + + default: + break; + } + + // re-enable signals, to avoid brute force exits. + // If brute force is needed, SIGKILL is available. + signal(sig, sigcatch); +} + + +/** + * Opens and reads /proc/sys/fs/mqueue/msg_max, to get the maximum number of allowed messages + * on POSIX MQ queues. rteval_parserd will use as much of this as possible when needed. + * + * @return Returns the system msg_max value, or DEFAULT_MSG_MAX on failure to read the setting. + */ +unsigned int get_mqueue_msg_max(LogContext *log) { + FILE *fp = NULL; + char buf[130]; + unsigned int msg_max = DEFAULT_MSG_MAX; + + fp = fopen("/proc/sys/fs/mqueue/msg_max", "r"); + if( !fp ) { + writelog(log, LOG_WARNING, + "Could not open /proc/sys/fs/mqueue/msg_max, defaulting to %i", + msg_max); + writelog(log, LOG_INFO, "%s", strerror(errno)); + return msg_max; + } + + memset(&buf, 0, 130); + if( fread(&buf, 1, 128, fp) < 1 ) { + writelog(log, LOG_WARNING, + "Could not read /proc/sys/fs/mqueue/msg_max, defaulting to %i", + msg_max); + writelog(log, LOG_INFO, "%s", strerror(errno)); + } else { + msg_max = atoi_nullsafe(buf); + if( msg_max < 1 ) { + msg_max = DEFAULT_MSG_MAX; + writelog(log, LOG_WARNING, + "Failed to parse /proc/sys/fs/mqueue/msg_max," + "defaulting to %i", msg_max); + } + } + fclose(fp); + return msg_max; +} + + +/** + * Main loop, which polls the submissionqueue table and puts jobs found here into a POSIX MQ queue + * which the worker threads will pick up. + * + * @param dbc Database connection, where to query the submission queue + * @param msgq file descriptor for the message queue + * + * @return Returns 0 on successful run, otherwise > 0 on errors. + */ +int process_submission_queue(dbconn *dbc, mqd_t msgq, int *activethreads) { + pthread_mutex_t mtx_submq = PTHREAD_MUTEX_INITIALIZER; + parseJob_t *job = NULL; + int rc = 0, i, actthr_cp = 0; + + while( shutdown == 0 ) { + // Check status if the worker threads + // If we don't have any worker threads, shut down immediately + writelog(dbc->log, LOG_DEBUG, "Active worker threads: %i", *activethreads); + if( *activethreads < 1 ) { + writelog(dbc->log, LOG_EMERG, + "All worker threads ceased to exist. Shutting down!"); + shutdown = 1; + rc = 1; + goto exit; + } + + if( db_ping(dbc) != 1 ) { + writelog(dbc->log, LOG_EMERG, "Lost connection to database. Shutting down!"); + shutdown = 1; + rc = 1; + goto exit; + } + + // Fetch an available job + job = db_get_submissionqueue_job(dbc, &mtx_submq); + if( !job ) { + writelog(dbc->log, LOG_EMERG, + "Failed to get submission queue job. Shutting down!"); + shutdown = 1; + rc = 1; + goto exit; + } + if( job->status == jbNONE ) { + free_nullsafe(job); + if( db_wait_notification(dbc, &shutdown, "rteval_submq") < 1 ) { + writelog(dbc->log, LOG_EMERG, + "Failed to wait for DB notification. Shutting down!"); + shutdown = 1; + rc = 1; + goto exit; + } + continue; + } + + // Send the job to the queue + writelog(dbc->log, LOG_INFO, "** New job: submid %i, %s", job->submid, job->filename); + do { + int res; + + errno = 0; + res = mq_send(msgq, (char *) job, sizeof(parseJob_t), 1); + if( (res < 0) && (errno != EAGAIN) ) { + writelog(dbc->log, LOG_EMERG, + "Could not send parse job to the queue. " + "Shutting down!"); + shutdown = 1; + rc = 2; + goto exit; + } else if( errno == EAGAIN ) { + writelog(dbc->log, LOG_WARNING, + "Message queue filled up. " + "Will not add new messages to queue for the next 60 seconds"); + sleep(60); + } + } while( (errno == EAGAIN) ); + free_nullsafe(job); + } + + exit: + // Send empty messages to the threads, to make them have a look at the shutdown flag + job = (parseJob_t *) malloc_nullsafe(dbc->log, sizeof(parseJob_t)); + errno = 0; + // Need to make a copy, as *activethreads will change when threads completes shutdown + actthr_cp = *activethreads; + for( i = 0; i < actthr_cp; i++ ) { + do { + int res; + + writelog(dbc->log, LOG_DEBUG, "%s shutdown message %i of %i", + (errno == EAGAIN ? "Resending" : "Sending"), i+1, *activethreads); + errno = 0; + res = mq_send(msgq, (char *) job, sizeof(parseJob_t), 1); + if( (res < 0) && (errno != EAGAIN) ) { + writelog(dbc->log, LOG_EMERG, + "Could not send parse job to the queue. " + "Shutting down!"); + shutdown = 1; + return rc; + } else if( errno == EAGAIN ) { + writelog(dbc->log, LOG_WARNING, + "Message queue filled up. " + "Will not add new messages to queue for the next 10 seconds"); + sleep(10); + } + } while( (errno == EAGAIN) ); + } + free_nullsafe(job); + return rc; +} + + +/** + * Prepares the program to be daemonised + * + * @param log Initialised log context, where log info of the process is reported + * + * @return Returns 1 on success, otherwise -1 + */ +int daemonise(LogContext *log) { + pid_t pid, sid; + int i = 0; + + if( (log->logtype == ltCONSOLE) ) { + writelog(log, LOG_EMERG, + "Cannot daemonise when logging to a console (stdout: or stderr:)"); + return -1; + } + + pid = fork(); + if (pid < 0) { + writelog(log, LOG_EMERG, "Failed to daemonise the process (fork)"); + return -1; + } else if (pid > 0) { + writelog(log, LOG_INFO, "Daemon pid: %ld", pid); + exit(EXIT_SUCCESS); + } + + umask(0); + + sid = setsid(); + if (sid < 0) { + writelog(log, LOG_EMERG, "Failed to daemonise the process (setsid)"); + return -1; + } + + if ((chdir("/")) < 0) { + writelog(log, LOG_EMERG, "Failed to daemonise the process (fork)"); + return -1; + } + + // Prepare stdin, stdout and stderr for daemon mode + close(2); + close(1); + close(0); + i = open("/dev/null", O_RDWR); /* open stdin */ + dup(i); /* stdout */ + dup(i); /* stderr */ + + writelog(log, LOG_INFO, "Daemonised successfully"); + return 1; +} + + +/** + * rtevald_parser main function. + * + * @param argc + * @param argv + * + * @return Returns the result of the process_submission_queue() function. + */ +int main(int argc, char **argv) { + eurephiaVALUES *config = NULL, *prgargs = NULL; + char xsltfile[2050], *reportdir = NULL; + xsltStylesheet *xslt = NULL; + dbconn *dbc = NULL; + pthread_t **threads = NULL; + pthread_attr_t **thread_attrs = NULL; + pthread_mutex_t mtx_sysreg = PTHREAD_MUTEX_INITIALIZER; + pthread_mutex_t mtx_thrcnt = PTHREAD_MUTEX_INITIALIZER; + threadData_t **thrdata = NULL; + struct mq_attr msgq_attr; + mqd_t msgq = 0; + int i,rc, mq_init = 0, max_threads = 0, started_threads = 0, activethreads = 0; + + // Initialise XML and XSLT libraries + xsltInit(); + xmlInitParser(); + + prgargs = parse_arguments(argc, argv); + if( prgargs == NULL ) { + fprintf(stderr, "** ERROR ** Failed to parse program arguments\n"); + rc = 2; + goto exit; + } + + // Setup a log context + logctx = init_log(eGet_value(prgargs, "log"), eGet_value(prgargs, "loglevel")); + if( !logctx ) { + fprintf(stderr, "** ERROR ** Could not setup a log context\n"); + eFree_values(prgargs); + rc = 2; + goto exit; + } + + // Fetch configuration + config = read_config(logctx, prgargs, "xmlrpc_parser"); + eFree_values(prgargs); // read_config() copies prgargs into config, we don't need prgargs anymore + + // Daemonise process if requested + if( atoi_nullsafe(eGet_value(config, "daemon")) == 1 ) { + if( daemonise(logctx) < 1 ) { + rc = 3; + goto exit; + } + } + + + // Parse XSLT template + snprintf(xsltfile, 512, "%s/%s", eGet_value(config, "xsltpath"), XMLPARSER_XSL); + writelog(logctx, LOG_DEBUG, "Parsing XSLT file: %s", xsltfile); + xslt = xsltParseStylesheetFile((xmlChar *) xsltfile); + if( !xslt ) { + writelog(logctx, LOG_EMERG, "Could not parse XSLT template: %s", xsltfile); + rc = 2; + goto exit; + } + + // Open a POSIX MQ + writelog(logctx, LOG_DEBUG, "Preparing POSIX MQ queue: /rteval_parsequeue"); + memset(&msgq, 0, sizeof(mqd_t)); + msgq_attr.mq_maxmsg = get_mqueue_msg_max(logctx); + msgq_attr.mq_msgsize = sizeof(parseJob_t); + msgq_attr.mq_flags = O_NONBLOCK; + msgq = mq_open("/rteval_parsequeue", O_RDWR | O_CREAT, 0600, &msgq_attr); + if( msgq < 0 ) { + writelog(logctx, LOG_EMERG, + "Could not open message queue: %s", strerror(errno)); + rc = 2; + goto exit; + } + mq_init = 1; + + // Get the number of worker threads + max_threads = atoi_nullsafe(eGet_value(config, "threads")); + if( max_threads == 0 ) { + max_threads = 4; + } + + // Get a database connection for the main thread + dbc = db_connect(config, max_threads, logctx); + if( !dbc ) { + rc = 4; + goto exit; + } + + // Prepare all threads + threads = calloc(max_threads + 1, sizeof(pthread_t *)); + thread_attrs = calloc(max_threads + 1, sizeof(pthread_attr_t *)); + thrdata = calloc(max_threads + 1, sizeof(threadData_t *)); + assert( (threads != NULL) && (thread_attrs != NULL) && (thrdata != NULL) ); + + reportdir = eGet_value(config, "reportdir"); + writelog(logctx, LOG_INFO, "Starting %i worker threads", max_threads); + for( i = 0; i < max_threads; i++ ) { + // Prepare thread specific data + thrdata[i] = malloc_nullsafe(logctx, sizeof(threadData_t)); + if( !thrdata[i] ) { + writelog(logctx, LOG_EMERG, + "Could not allocate memory for thread data"); + rc = 2; + goto exit; + } + + // Get a database connection for the thread + thrdata[i]->dbc = db_connect(config, i, logctx); + if( !thrdata[i]->dbc ) { + writelog(logctx, LOG_EMERG, + "Could not connect to the database for thread %i", i); + rc = 2; + shutdown = 1; + goto exit; + } + + thrdata[i]->shutdown = &shutdown; + thrdata[i]->threadcount = &activethreads; + thrdata[i]->mtx_thrcnt = &mtx_thrcnt; + thrdata[i]->id = i; + thrdata[i]->msgq = msgq; + thrdata[i]->mtx_sysreg = &mtx_sysreg; + thrdata[i]->xslt = xslt; + thrdata[i]->destdir = reportdir; + + thread_attrs[i] = malloc_nullsafe(logctx, sizeof(pthread_attr_t)); + if( !thread_attrs[i] ) { + writelog(logctx, LOG_EMERG, + "Could not allocate memory for thread attributes"); + rc = 2; + goto exit; + } + pthread_attr_init(thread_attrs[i]); + pthread_attr_setdetachstate(thread_attrs[i], PTHREAD_CREATE_JOINABLE); + + threads[i] = malloc_nullsafe(logctx, sizeof(pthread_t)); + if( !threads[i] ) { + writelog(logctx, LOG_EMERG, + "Could not allocate memory for pthread_t"); + rc = 2; + goto exit; + } + } + + // Setup signal catching + signal(SIGINT, sigcatch); + signal(SIGTERM, sigcatch); + signal(SIGHUP, SIG_IGN); + signal(SIGUSR1, sigcatch); + signal(SIGUSR2, SIG_IGN); + + // Start the threads + for( i = 0; i < max_threads; i++ ) { + int thr_rc = pthread_create(threads[i], thread_attrs[i], parsethread, thrdata[i]); + if( thr_rc < 0 ) { + writelog(logctx, LOG_EMERG, + "** ERROR ** Failed to start thread %i: %s", + i, strerror(thr_rc)); + rc = 3; + goto exit; + } + started_threads++; + } + + // Main routine + // + // checks the submission queue and puts unprocessed records on the POSIX MQ + // to be parsed by one of the threads + // + writelog(logctx, LOG_DEBUG, "Starting submission queue checker"); + rc = process_submission_queue(dbc, msgq, &activethreads); + writelog(logctx, LOG_DEBUG, "Submission queue checker shut down"); + + exit: + // Clean up all threads + for( i = 0; i < max_threads; i++ ) { + // Wait for all threads to exit + if( (i < started_threads) && threads && threads[i] ) { + void *thread_rc; + int j_rc; + + if( (j_rc = pthread_join(*threads[i], &thread_rc)) != 0 ) { + writelog(logctx, LOG_CRIT, + "Failed to join thread %i: %s", + i, strerror(j_rc)); + } + pthread_attr_destroy(thread_attrs[i]); + } + free_nullsafe(threads[i]); + free_nullsafe(thread_attrs[i]); + + // Disconnect threads database connection + if( thrdata && thrdata[i] ) { + db_disconnect(thrdata[i]->dbc); + free_nullsafe(thrdata[i]); + } + } + free_nullsafe(thrdata); + free_nullsafe(threads); + free_nullsafe(thread_attrs); + + // Close message queue + if( mq_init == 1 ) { + errno = 0; + if( mq_close(msgq) < 0 ) { + writelog(logctx, LOG_CRIT, "Failed to close message queue: %s", + strerror(errno)); + } + errno = 0; + if( mq_unlink("/rteval_parsequeue") < 0 ) { + writelog(logctx, LOG_ALERT, "Failed to remove the message queue: %s", + strerror(errno)); + } + } + + // Disconnect from database, main thread connection + db_disconnect(dbc); + + // Free up the rest + eFree_values(config); + xsltFreeStylesheet(xslt); + xmlCleanupParser(); + xsltCleanupGlobals(); + + writelog(logctx, LOG_EMERG, "rteval_parserd is stopped"); + close_log(logctx); + return rc; +} + diff --git a/server/parser/sha1.c b/server/parser/sha1.c new file mode 100644 index 0000000..3f77aa9 --- /dev/null +++ b/server/parser/sha1.c @@ -0,0 +1,615 @@ +/*- + * Copyright (c) 2001-2003 Allan Saddi <allan@saddi.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ALLAN SADDI AND HIS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ALLAN SADDI OR HIS CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $Id: sha1.c 680 2003-07-25 21:57:38Z asaddi $ + */ + +/* + * Define WORDS_BIGENDIAN if compiling on a big-endian architecture. + * + * Define SHA1_TEST to test the implementation using the NIST's + * sample messages. The output should be: + * + * a9993e36 4706816a ba3e2571 7850c26c 9cd0d89d + * 84983e44 1c3bd26e baae4aa1 f95129e5 e54670f1 + * 34aa973c d4c4daa4 f61eeb2b dbad2731 6534016f + */ + +#include <stdint.h> +#include <string.h> + +#include "sha1.h" + +#ifndef lint +static const char rcsid[] = + "$Id: sha1.c 680 2003-07-25 21:57:38Z asaddi $"; +#endif /* !lint */ + +#define ROTL(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) +#define ROTR(x, n) (((x) >> (n)) | ((x) << (32 - (n)))) + +#define F_0_19(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) +#define F_20_39(x, y, z) ((x) ^ (y) ^ (z)) +#define F_40_59(x, y, z) (((x) & ((y) | (z))) | ((y) & (z))) +#define F_60_79(x, y, z) ((x) ^ (y) ^ (z)) + +#define DO_ROUND(F, K) { \ + temp = ROTL(a, 5) + F(b, c, d) + e + *(W++) + K; \ + e = d; \ + d = c; \ + c = ROTL(b, 30); \ + b = a; \ + a = temp; \ +} + +#define K_0_19 0x5a827999L +#define K_20_39 0x6ed9eba1L +#define K_40_59 0x8f1bbcdcL +#define K_60_79 0xca62c1d6L + +#ifndef RUNTIME_ENDIAN + +#ifdef WORDS_BIGENDIAN + +#define BYTESWAP(x) (x) +#define BYTESWAP64(x) (x) + +#else /* WORDS_BIGENDIAN */ + +#define BYTESWAP(x) ((ROTR((x), 8) & 0xff00ff00L) | \ + (ROTL((x), 8) & 0x00ff00ffL)) +#define BYTESWAP64(x) _byteswap64(x) + +static inline uint64_t _byteswap64(uint64_t x) +{ + uint32_t a = x >> 32; + uint32_t b = (uint32_t) x; + return ((uint64_t) BYTESWAP(b) << 32) | (uint64_t) BYTESWAP(a); +} + +#endif /* WORDS_BIGENDIAN */ + +#else /* !RUNTIME_ENDIAN */ + +#define BYTESWAP(x) _byteswap(sc->littleEndian, x) +#define BYTESWAP64(x) _byteswap64(sc->littleEndian, x) + +#define _BYTESWAP(x) ((ROTR((x), 8) & 0xff00ff00L) | \ + (ROTL((x), 8) & 0x00ff00ffL)) +#define _BYTESWAP64(x) __byteswap64(x) + +static inline uint64_t __byteswap64(uint64_t x) +{ + uint32_t a = x >> 32; + uint32_t b = (uint32_t) x; + return ((uint64_t) _BYTESWAP(b) << 32) | (uint64_t) _BYTESWAP(a); +} + +static inline uint32_t _byteswap(int littleEndian, uint32_t x) +{ + if (!littleEndian) + return x; + else + return _BYTESWAP(x); +} + +static inline uint64_t _byteswap64(int littleEndian, uint64_t x) +{ + if (!littleEndian) + return x; + else + return _BYTESWAP64(x); +} + +static inline void setEndian(int *littleEndianp) +{ + union { + uint32_t w; + uint8_t b[4]; + } endian; + + endian.w = 1L; + *littleEndianp = endian.b[0] != 0; +} + +#endif /* !RUNTIME_ENDIAN */ + +static const uint8_t padding[64] = { + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +void +SHA1Init (SHA1Context *sc) +{ +#ifdef RUNTIME_ENDIAN + setEndian (&sc->littleEndian); +#endif /* RUNTIME_ENDIAN */ + + sc->totalLength = 0LL; + sc->hash[0] = 0x67452301L; + sc->hash[1] = 0xefcdab89L; + sc->hash[2] = 0x98badcfeL; + sc->hash[3] = 0x10325476L; + sc->hash[4] = 0xc3d2e1f0L; + sc->bufferLength = 0L; +} + +static void +burnStack (int size) +{ + char buf[128]; + + memset (buf, 0, sizeof (buf)); + size -= sizeof (buf); + if (size > 0) + burnStack (size); +} + +static void +SHA1Guts (SHA1Context *sc, const uint32_t *cbuf) +{ + uint32_t buf[80]; + uint32_t *W, *W3, *W8, *W14, *W16; + uint32_t a, b, c, d, e, temp; + int i; + + W = buf; + + for (i = 15; i >= 0; i--) { + *(W++) = BYTESWAP(*cbuf); + cbuf++; + } + + W16 = &buf[0]; + W14 = &buf[2]; + W8 = &buf[8]; + W3 = &buf[13]; + + for (i = 63; i >= 0; i--) { + *W = *(W3++) ^ *(W8++) ^ *(W14++) ^ *(W16++); + *W = ROTL(*W, 1); + W++; + } + + a = sc->hash[0]; + b = sc->hash[1]; + c = sc->hash[2]; + d = sc->hash[3]; + e = sc->hash[4]; + + W = buf; + +#ifndef SHA1_UNROLL +#define SHA1_UNROLL 20 +#endif /* !SHA1_UNROLL */ + +#if SHA1_UNROLL == 1 + for (i = 19; i >= 0; i--) + DO_ROUND(F_0_19, K_0_19); + + for (i = 19; i >= 0; i--) + DO_ROUND(F_20_39, K_20_39); + + for (i = 19; i >= 0; i--) + DO_ROUND(F_40_59, K_40_59); + + for (i = 19; i >= 0; i--) + DO_ROUND(F_60_79, K_60_79); +#elif SHA1_UNROLL == 2 + for (i = 9; i >= 0; i--) { + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + } + + for (i = 9; i >= 0; i--) { + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + } + + for (i = 9; i >= 0; i--) { + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + } + + for (i = 9; i >= 0; i--) { + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + } +#elif SHA1_UNROLL == 4 + for (i = 4; i >= 0; i--) { + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + } + + for (i = 4; i >= 0; i--) { + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + } + + for (i = 4; i >= 0; i--) { + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + } + + for (i = 4; i >= 0; i--) { + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + } +#elif SHA1_UNROLL == 5 + for (i = 3; i >= 0; i--) { + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + } + + for (i = 3; i >= 0; i--) { + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + } + + for (i = 3; i >= 0; i--) { + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + } + + for (i = 3; i >= 0; i--) { + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + } +#elif SHA1_UNROLL == 10 + for (i = 1; i >= 0; i--) { + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + } + + for (i = 1; i >= 0; i--) { + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + } + + for (i = 1; i >= 0; i--) { + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + } + + for (i = 1; i >= 0; i--) { + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + } +#elif SHA1_UNROLL == 20 + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + DO_ROUND(F_0_19, K_0_19); + + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + DO_ROUND(F_20_39, K_20_39); + + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + DO_ROUND(F_40_59, K_40_59); + + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); + DO_ROUND(F_60_79, K_60_79); +#else /* SHA1_UNROLL */ +#error SHA1_UNROLL must be 1, 2, 4, 5, 10 or 20! +#endif + + sc->hash[0] += a; + sc->hash[1] += b; + sc->hash[2] += c; + sc->hash[3] += d; + sc->hash[4] += e; +} + +void +SHA1Update (SHA1Context *sc, const void *vdata, uint32_t len) +{ + const uint8_t *data = vdata; + uint32_t bufferBytesLeft; + uint32_t bytesToCopy; + int needBurn = 0; + +#ifdef SHA1_FAST_COPY + if (sc->bufferLength) { + bufferBytesLeft = 64L - sc->bufferLength; + + bytesToCopy = bufferBytesLeft; + if (bytesToCopy > len) + bytesToCopy = len; + + memcpy (&sc->buffer.bytes[sc->bufferLength], data, bytesToCopy); + + sc->totalLength += bytesToCopy * 8L; + + sc->bufferLength += bytesToCopy; + data += bytesToCopy; + len -= bytesToCopy; + + if (sc->bufferLength == 64L) { + SHA1Guts (sc, sc->buffer.words); + needBurn = 1; + sc->bufferLength = 0L; + } + } + + while (len > 63) { + sc->totalLength += 512L; + + SHA1Guts (sc, data); + needBurn = 1; + + data += 64L; + len -= 64L; + } + + if (len) { + memcpy (&sc->buffer.bytes[sc->bufferLength], data, len); + + sc->totalLength += len * 8L; + + sc->bufferLength += len; + } +#else /* SHA1_FAST_COPY */ + while (len) { + bufferBytesLeft = 64L - sc->bufferLength; + + bytesToCopy = bufferBytesLeft; + if (bytesToCopy > len) + bytesToCopy = len; + + memcpy (&sc->buffer.bytes[sc->bufferLength], data, bytesToCopy); + + sc->totalLength += bytesToCopy * 8L; + + sc->bufferLength += bytesToCopy; + data += bytesToCopy; + len -= bytesToCopy; + + if (sc->bufferLength == 64L) { + SHA1Guts (sc, sc->buffer.words); + needBurn = 1; + sc->bufferLength = 0L; + } + } +#endif /* SHA1_FAST_COPY */ + + if (needBurn) + burnStack (sizeof (uint32_t[86]) + sizeof (uint32_t *[5]) + sizeof (int)); +} + +void +SHA1Final (SHA1Context *sc, uint8_t hash[SHA1_HASH_SIZE]) +{ + uint32_t bytesToPad; + uint64_t lengthPad; + int i; + + bytesToPad = 120L - sc->bufferLength; + if (bytesToPad > 64L) + bytesToPad -= 64L; + + lengthPad = BYTESWAP64(sc->totalLength); + + SHA1Update (sc, padding, bytesToPad); + SHA1Update (sc, &lengthPad, 8L); + + if (hash) { + for (i = 0; i < SHA1_HASH_WORDS; i++) { +#ifdef SHA1_FAST_COPY + *((uint32_t *) hash) = BYTESWAP(sc->hash[i]); +#else /* SHA1_FAST_COPY */ + hash[0] = (uint8_t) (sc->hash[i] >> 24); + hash[1] = (uint8_t) (sc->hash[i] >> 16); + hash[2] = (uint8_t) (sc->hash[i] >> 8); + hash[3] = (uint8_t) sc->hash[i]; +#endif /* SHA1_FAST_COPY */ + hash += 4; + } + } +} + +#ifdef SHA1_TEST + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +int +main (int argc, char *argv[]) +{ + SHA1Context foo; + uint8_t hash[SHA1_HASH_SIZE]; + char buf[1000]; + int i; + + SHA1Init (&foo); + SHA1Update (&foo, "abc", 3); + SHA1Final (&foo, hash); + + for (i = 0; i < SHA1_HASH_SIZE;) { + printf ("%02x", hash[i++]); + if (!(i % 4)) + printf (" "); + } + printf ("\n"); + + SHA1Init (&foo); + SHA1Update (&foo, + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + 56); + SHA1Final (&foo, hash); + + for (i = 0; i < SHA1_HASH_SIZE;) { + printf ("%02x", hash[i++]); + if (!(i % 4)) + printf (" "); + } + printf ("\n"); + + SHA1Init (&foo); + memset (buf, 'a', sizeof (buf)); + for (i = 0; i < 1000; i++) + SHA1Update (&foo, buf, sizeof (buf)); + SHA1Final (&foo, hash); + + for (i = 0; i < SHA1_HASH_SIZE;) { + printf ("%02x", hash[i++]); + if (!(i % 4)) + printf (" "); + } + printf ("\n"); + + exit (0); +} + +#endif /* SHA1_TEST */ diff --git a/server/parser/sha1.h b/server/parser/sha1.h new file mode 100644 index 0000000..9ce5bd6 --- /dev/null +++ b/server/parser/sha1.h @@ -0,0 +1,66 @@ +/*- + * Copyright (c) 2001-2003 Allan Saddi <allan@saddi.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY ALLAN SADDI AND HIS CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL ALLAN SADDI OR HIS CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $Id: sha1.h 347 2003-02-23 22:11:49Z asaddi $ + */ + +#ifndef _SHA1_H +#define _SHA1_H + +#include <stdint.h> + +#define SHA1_HASH_SIZE 20 + +/* Hash size in 32-bit words */ +#define SHA1_HASH_WORDS 5 + +struct _SHA1Context { + uint64_t totalLength; + uint32_t hash[SHA1_HASH_WORDS]; + uint32_t bufferLength; + union { + uint32_t words[16]; + uint8_t bytes[64]; + } buffer; +#ifdef RUNTIME_ENDIAN + int littleEndian; +#endif /* RUNTIME_ENDIAN */ +}; + +typedef struct _SHA1Context SHA1Context; + +#ifdef __cplusplus +extern "C" { +#endif + +void SHA1Init (SHA1Context *sc); +void SHA1Update (SHA1Context *sc, const void *data, uint32_t len); +void SHA1Final (SHA1Context *sc, uint8_t hash[SHA1_HASH_SIZE]); + +#ifdef __cplusplus +} +#endif + +#endif /* _SHA1_H */ diff --git a/server/parser/statuses.h b/server/parser/statuses.h new file mode 100644 index 0000000..701c8e8 --- /dev/null +++ b/server/parser/statuses.h @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file statuses.h + * @author David Sommerseth <davids@redhat.com> + * @date Wed Oct 21 11:17:24 2009 + * + * @brief Status values used by rteval_parserd + * + */ + +#ifndef _RTEVAL_STATUS_H +#define _RTEVAL_STATUS_H + +#define STAT_NEW 0 /**< New, unparsed report in the submission queue */ +#define STAT_ASSIGNED 1 /**< Submission is assigned to a parser */ +#define STAT_INPROG 2 /**< Parsing has started */ +#define STAT_SUCCESS 3 /**< Report parsed successfully */ +#define STAT_UNKNFAIL 4 /**< Unkown failure */ +#define STAT_XMLFAIL 5 /**< Failed to parse the report XML file */ +#define STAT_SYSREG 6 /**< System registration failed */ +#define STAT_RTERIDREG 7 /**< Failed to get a new rterid value for the rteval run */ +#define STAT_GENDB 8 /**< General database error */ +#define STAT_RTEVRUNS 9 /**< Registering rteval run information failed */ +#define STAT_CYCLIC 10 /**< Registering cyclictest results failed */ +#define STAT_REPMOVE 11 /**< Failed to move the report file */ + +#endif diff --git a/server/parser/threadinfo.h b/server/parser/threadinfo.h new file mode 100644 index 0000000..d2bcf43 --- /dev/null +++ b/server/parser/threadinfo.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file threadinfo.h + * @author David Sommerseth <davids@redhat.com> + * @date Thu Oct 15 11:47:51 2009 + * + * @brief Shared info between the main() and parsethread() functions + * + */ + +#ifndef _THREADINFO_H +#define _THREADINFO_H + +#include <mqueue.h> +#include <libxslt/transform.h> + +/** + * Thread slot information. Each thread slot is assigned with one threadData_t element. + */ +typedef struct { + int *shutdown; /**< If set to 1, the thread should shut down */ + int *threadcount; /**< Number of active worker threads */ + pthread_mutex_t *mtx_thrcnt; /**< Mutex lock for updating active worker threads */ + mqd_t msgq; /**< POSIX MQ descriptor */ + pthread_mutex_t *mtx_sysreg; /**< Mutex locking, to avoid clashes with registering systems */ + unsigned int id; /**< Numeric ID for this thread */ + dbconn *dbc; /**< Database connection assigned to this thread */ + xsltStylesheet *xslt; /**< XSLT stylesheet assigned to this thread */ + const char *destdir; /**< Directory where to put the parsed reports */ +} threadData_t; + +#endif diff --git a/server/parser/xmlparser.c b/server/parser/xmlparser.c new file mode 100644 index 0000000..8cf13a8 --- /dev/null +++ b/server/parser/xmlparser.c @@ -0,0 +1,414 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * David Sommerseth <davids@redhat.com> + * + * Parses summary.xml reports from rteval into a standardised XML format + * which is useful when putting data into a database. + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file xmlparser.c + * @author David Sommerseth <davids@redhat.com> + * @date Wed Oct 21 10:58:53 2009 + * + * @brief Parses summary.xml reports from rteval into a standardised XML format + * which is useful when putting data into a database. + * + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include <libxml/tree.h> +#include <libxslt/xsltInternals.h> +#include <libxslt/transform.h> +#include <libxslt/xsltutils.h> + +#include <eurephia_nullsafe.h> +#include <eurephia_xml.h> +#include <xmlparser.h> +#include <sha1.h> +#include <log.h> + +/** + * Simple strdup() function which encapsulates the string in single quotes, + * which is needed for XSLT parameter values + * + * @param str The string to be strdup'ed and encapsulated + * + * @return Returns a pointer to the new buffer. + */ +static char *encapsString(const char *str) { + char *ret = NULL; + + if( str == NULL ) { + return NULL; + } + + ret = (char *) calloc(1, strlen(str)+4); + assert( ret != NULL ); + + snprintf(ret, strlen(str)+3, "'%s'", str); + return ret; +} + + +/** + * Converts an integer to string an encapsulates the value in single quotes, + * which is needed for XSLT parameter values. + * + * @param val Integer value to encapsulate + * + * @return Returns a pointer to a new buffer with the encapsulated integer value. This + * buffer must be free'd after usage. + */ +static char *encapsInt(const unsigned int val) { + char *buf = NULL; + + buf = (char *) calloc(1, 130); + snprintf(buf, 128, "'%i'", val); + return buf; +} + + +/** + * Parses any XML input document into a sqldata XML format which can be used by pgsql_INSERT(). + * The transformation must be defined in the input XSLT template. + * + * @param log Log context + * @param xslt XSLT template defining the data transformation + * @param indata_d Input XML data to transform to a sqldata XML document + * @param params Parameters to be sent to the XSLT parser + * + * @return Returns a well formed sqldata XML document on success, otherwise NULL is returned. + */ +xmlDoc *parseToSQLdata(LogContext *log, xsltStylesheet *xslt, xmlDoc *indata_d, parseParams *params) { + xmlDoc *result_d = NULL; + char *xsltparams[10]; + unsigned int idx = 0, idx_table = 0, idx_submid = 0, + idx_syskey = 0, idx_rterid = 0, idx_repfname = 0; + + if( params->table == NULL ) { + writelog(log, LOG_ERR, "Table is not defined"); + return NULL; + } + + // Prepare XSLT parameters + xsltparams[idx++] = "table\0"; + xsltparams[idx] = (char *) encapsString(params->table); + idx_table = idx++; + + if( params->submid > 0) { + xsltparams[idx++] = "submid\0"; + xsltparams[idx] = (char *) encapsInt(params->submid); + idx_submid = idx++; + } + + if( params->syskey > 0) { + xsltparams[idx++] = "syskey\0"; + xsltparams[idx] = (char *) encapsInt(params->syskey); + idx_syskey = idx++; + } + + if( params->rterid > 0 ) { + xsltparams[idx++] = "rterid"; + xsltparams[idx] = (char *) encapsInt(params->rterid); + idx_rterid = idx++; + } + + if( params->report_filename ) { + xsltparams[idx++] = "report_filename"; + xsltparams[idx] = (char *) encapsString(params->report_filename); + idx_repfname = idx++; + } + xsltparams[idx] = NULL; + + // Apply the XSLT template to the input XML data + result_d = xsltApplyStylesheet(xslt, indata_d, (const char **)xsltparams); + if( result_d == NULL ) { + writelog(log, LOG_CRIT, "Failed applying XSLT template to input XML"); + } + + // Free memory we allocated via encapsString()/encapsInt() + free(xsltparams[idx_table]); + if( params->submid ) { + free(xsltparams[idx_submid]); + } + if( params->syskey ) { + free(xsltparams[idx_syskey]); + } + if( params->rterid ) { + free(xsltparams[idx_rterid]); + } + if( params->report_filename ) { + free(xsltparams[idx_repfname]); + } + + return result_d; +} + + +/** + * Internal xmlparser function. Extracts the value from a '//sqldata/records/record/value' + * node and hashes the value if the 'hash' attribute is set. Otherwise the value is extracted + * from the node directly. This function is only used by sqldataExtractContent(). + * + * @param sql_n sqldata values node containing the value to extract. + * + * @return Returns a pointer to a new buffer containing the value on success, otherwise NULL. + * This memory buffer must be free'd after usage. + */ +static inline char *sqldataValueHash(LogContext *log, xmlNode *sql_n) { + const char *hash = NULL; + SHA1Context shactx; + uint8_t shahash[SHA1_HASH_SIZE]; + char *ret = NULL, *ptr = NULL;; + int i; + + if( !sql_n || (xmlStrcmp(sql_n->name, (xmlChar *) "value") != 0) + || (xmlStrcmp(sql_n->parent->name, (xmlChar *) "record") != 0) ) { + return NULL; + } + + hash = xmlGetAttrValue(sql_n->properties, "hash"); + if( !hash ) { + // If no hash attribute is found, just use the raw data + ret = strdup_nullsafe(xmlExtractContent(sql_n)); + } else if( strcasecmp(hash, "sha1") == 0 ) { + const char *indata = xmlExtractContent(sql_n); + // SHA1 hashing requested + SHA1Init(&shactx); + SHA1Update(&shactx, indata, strlen_nullsafe(indata)); + SHA1Final(&shactx, shahash); + + // "Convert" to a readable format + ret = malloc_nullsafe(log, (SHA1_HASH_SIZE * 2) + 3); + ptr = ret; + for( i = 0; i < SHA1_HASH_SIZE; i++ ) { + sprintf(ptr, "%02x", shahash[i]); + ptr += 2; + } + } else { + ret = strdup("<Unsupported hashing algorithm>"); + } + + return ret; +} + + +/** + * Extract the content of a '//sqldata/records/record/value' node. It will consider + * both the 'hash' and 'type' attributes of the 'value' tag. + * + * @param log Log context + * @param sql_n Pointer to a value node of a sqldata XML document. + * + * @return Returns a pointer to a new memory buffer containing the value as a string. + * On errors, NULL is returned. This memory buffer must be free'd after usage. + */ +char *sqldataExtractContent(LogContext *log, xmlNode *sql_n) { + const char *valtype = xmlGetAttrValue(sql_n->properties, "type"); + + if( !sql_n || (xmlStrcmp(sql_n->name, (xmlChar *) "value") != 0) + || (xmlStrcmp(sql_n->parent->name, (xmlChar *) "record") != 0) ) { + return NULL; + } + + if( valtype && (strcmp(valtype, "xmlblob") == 0) ) { + xmlNode *chld_n = sql_n->children; + + // Go to next "real" tag, skipping non-element nodes + while( chld_n && chld_n->type != XML_ELEMENT_NODE ){ + chld_n = chld_n->next; + } + return xmlNodeToString(log, chld_n); + } else { + return sqldataValueHash(log, sql_n); + } +} + + +/** + * Return the 'fid' value of a given field in an sqldata XML document. + * + * @param log Log context + * @param sql_n Pointer to the root xmlNode element of a sqldata XML document + * @param fname String containing the field name to look up + * + * @return Returns a value >= 0 on success, containing the 'fid' value of the field. Otherwise + * a value < 0 is returned. -1 if the field is not found or -2 if there are some problems + * with the XML document. + */ +int sqldataGetFid(LogContext *log, xmlNode *sql_n, const char *fname) { + xmlNode *f_n = NULL; + + if( !sql_n || (xmlStrcmp(sql_n->name, (xmlChar *) "sqldata") != 0) ) { + writelog(log, LOG_ERR, + "sqldataGetFid: Input XML document is not a valid sqldata document"); + return -2; + } + + f_n = xmlFindNode(sql_n, "fields"); + if( !f_n || !f_n->children ) { + writelog(log, LOG_ERR, + "sqldataGetFid: Input XML document does not contain a fields section"); + return -2; + } + + foreach_xmlnode(f_n->children, f_n) { + if( (f_n->type != XML_ELEMENT_NODE) + || xmlStrcmp(f_n->name, (xmlChar *) "field") != 0 ) { + // Skip uninteresting nodes + continue; + } + + if( strcmp(xmlExtractContent(f_n), fname) == 0 ) { + char *fid = xmlGetAttrValue(f_n->properties, "fid"); + if( !fid ) { + writelog(log, LOG_ERR, + "sqldataGetFid: Field node is missing 'fid' attribute (field: %s)", + fname); + return -2; + } + return atoi_nullsafe(fid); + } + } + return -1; +} + + +/** + * Retrieves the value of a particular field in an sqldata XML document. + * + * @param log Log context + * @param sqld pointer to an sqldata XML document. + * @param fname String containing the field name to extract the value of. + * @param recid Integer containing the record ID of the record to extract the value. This starts + * on 0. + * + * @return Returns a pointer to a new memory buffer containing the extracted value. On errors or if + * recid is higher than available records, NULL is returned. + */ +char *sqldataGetValue(LogContext *log, xmlDoc *sqld, const char *fname, int recid ) { + xmlNode *r_n = NULL; + int fid = -3, rc = 0; + + if( recid < 0 ) { + writelog(log, LOG_ERR, "sqldataGetValue: Invalid recid"); + return NULL; + } + + r_n = xmlDocGetRootElement(sqld); + if( !r_n || (xmlStrcmp(r_n->name, (xmlChar *) "sqldata") != 0) ) { + writelog(log, LOG_ERR, + "sqldataGetValue: Input XML document is not a valid sqldata document"); + return NULL; + } + + fid = sqldataGetFid(log, r_n, fname); + if( fid < 0 ) { + return NULL; + } + + r_n = xmlFindNode(r_n, "records"); + if( !r_n || !r_n->children ) { + writelog(log, LOG_ERR, + "sqldataGetValue: Input XML document does not contain a records section"); + return NULL; + } + + foreach_xmlnode(r_n->children, r_n) { + if( (r_n->type != XML_ELEMENT_NODE) + || xmlStrcmp(r_n->name, (xmlChar *) "record") != 0 ) { + // Skip uninteresting nodes + continue; + } + if( rc == recid ) { + xmlNode *v_n = NULL; + // The rigth record is found, find the field we're looking for + foreach_xmlnode(r_n->children, v_n) { + char *fid_s = NULL; + if( (v_n->type != XML_ELEMENT_NODE) + || (xmlStrcmp(v_n->name, (xmlChar *) "value") != 0) ) { + // Skip uninteresting nodes + continue; + } + fid_s = xmlGetAttrValue(v_n->properties, "fid"); + if( fid_s && (fid == atoi_nullsafe(fid_s)) ) { + return sqldataExtractContent(log, v_n); + } + } + } + rc++; + } + return NULL; +} + + +/** + * Helper function to parse an sqldata XML document for the systems_hostname table. In addition + * it will also return two strings containing hostname and ipaddress of the host. + * + * @param log Log context + * @param xslt Pointer to an xmlparser.xml XSLT template + * @param summaryxml rteval XML report document + * @param syskey Integer containing the syskey value corresponding to this host + * @param hostname Return pointer for where the hostname will be saved. + * @param ipaddr Return pointer for where the IP address will be saved. + * + * @return Returns a sqldata XML document on success. In this case the hostname and ipaddr will point + * at memory buffers containing hostname and ipaddress. These values must be free'd after usage. + * On errors the function will return NULL and hostname and ipaddr will not have been touched + * at all. + */ +xmlDoc *sqldataGetHostInfo(LogContext *log, xsltStylesheet *xslt, xmlDoc *summaryxml, + int syskey, char **hostname, char **ipaddr) +{ + xmlDoc *hostinfo_d = NULL; + parseParams prms; + + memset(&prms, 0, sizeof(parseParams)); + prms.table = "systems_hostname"; + prms.syskey = syskey; + + hostinfo_d = parseToSQLdata(log, xslt, summaryxml, &prms); + if( !hostinfo_d ) { + writelog(log, LOG_ERR, + "sqldatGetHostInfo: Could not parse input XML data"); + xmlFreeDoc(hostinfo_d); + goto exit; + } + + // Grab hostname from input XML + *hostname = sqldataGetValue(log, hostinfo_d, "hostname", 0); + if( !hostname ) { + writelog(log, LOG_ERR, + "sqldatGetHostInfo: Could not retrieve the hostname field from the input XML"); + xmlFreeDoc(hostinfo_d); + goto exit; + } + + // Grab ipaddr from input XML + *ipaddr = sqldataGetValue(log, hostinfo_d, "ipaddr", 0); + if( !ipaddr ) { + writelog(log, LOG_ERR, + "sqldatGetHostInfo: Could not retrieve the IP address field from the input XML"); + free_nullsafe(hostname); + xmlFreeDoc(hostinfo_d); + goto exit; + } + exit: + return hostinfo_d; +} diff --git a/server/parser/xmlparser.h b/server/parser/xmlparser.h new file mode 100644 index 0000000..2c96fdf --- /dev/null +++ b/server/parser/xmlparser.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2009 Red Hat Inc. + * + * David Sommerseth <davids@redhat.com> + * + * + * This application is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2. + * + * This application is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/** + * @file xmlparser.h + * @author David Sommerseth <davids@redhat.com> + * @date Wed Oct 7 17:27:39 2009 + * + * @brief Parses summary.xml reports from rteval into a standardised XML format + * which is useful when putting data into a database. + * + */ + + +#ifndef _XMLPARSER_H +#define _XMLPARSER_H + +/** + * Parameters needed by the the xmlparser.xsl XSLT template. + */ +typedef struct { + const char *table; /**< Which table to parse data for. Required*/ + unsigned int submid; /**< Submission ID, needed by the 'rtevalruns' table */ + unsigned int syskey; /**< System key (referencing systems.syskey) */ + const char *report_filename; /**< Filename to the saved report (after being parsed) */ + unsigned int rterid; /**< References rtevalruns.rterid */ +} parseParams; + +xmlDoc *parseToSQLdata(LogContext *log, xsltStylesheet *xslt, xmlDoc *indata_d, parseParams *params); +char *sqldataExtractContent(LogContext *log, xmlNode *sql_n); +int sqldataGetFid(LogContext *log, xmlNode *sqld, const char *fname); +char *sqldataGetValue(LogContext *log, xmlDoc *sqld, const char *fname, int recid); +xmlDoc *sqldataGetHostInfo(LogContext *log, xsltStylesheet *xslt, xmlDoc *summaryxml, + int syskey, char **hostname, char **ipaddr); +#endif diff --git a/server/xmlparser.xsl b/server/parser/xmlparser.xsl index cf3483b..0313a67 100644 --- a/server/xmlparser.xsl +++ b/server/parser/xmlparser.xsl @@ -47,6 +47,11 @@ <xsl:text>Invalid 'syskey' parameter value: </xsl:text><xsl:value-of select="syskey"/> </xsl:message> </xsl:if> + <xsl:if test="string(number($rterid)) = 'NaN'"> + <xsl:message terminate="yes"> + <xsl:text>Invalid rterid' parameter value: </xsl:text><xsl:value-of select="$rterid"/> + </xsl:message> + </xsl:if> <xsl:if test="$report_filename = ''"> <xsl:message terminate="yes"> <xsl:text>The parameter 'report_filename' parameter cannot be empty</xsl:text> @@ -85,6 +90,29 @@ <xsl:apply-templates select="/rteval/cyclictest/RawSampleData" mode="cyclic_raw_sql"/> </xsl:when> + <!-- TABLE: cyclic_histogram --> + <xsl:when test="$table = 'cyclic_histogram'"> + <xsl:if test="string(number($rterid)) = 'NaN'"> + <xsl:message terminate="yes"> + <xsl:text>Invalid 'rterid' parameter value: </xsl:text><xsl:value-of select="$rterid"/> + </xsl:message> + </xsl:if> + <sqldata table="cyclic_histogram"> + <fields> + <field fid="0">rterid</field> + <field fid="1">core</field> + <field fid="2">index</field> + <field fid="3">value</field> + </fields> + <records> + <xsl:apply-templates select="/rteval/cyclictest/system/histogram/bucket" + mode="cyclic_histogram_sql"/> + <xsl:apply-templates select="/rteval/cyclictest/core/histogram/bucket" + mode="cyclic_histogram_sql"/> + </records> + </sqldata> + </xsl:when> + <xsl:otherwise> <xsl:message terminate="yes"> <xsl:text>Invalid 'table' parameter value: </xsl:text><xsl:value-of select="$table"/> @@ -130,7 +158,7 @@ </xsl:template> <xsl:template match="/rteval" mode="rtevalruns_sql"> - <sqldata table="rtevalruns" key="rterid"> + <sqldata table="rtevalruns"> <fields> <field fid="0">syskey</field> <field fid="1">kernel_ver</field> @@ -141,6 +169,8 @@ <field fid="6">load_avg</field> <field fid="7">version</field> <field fid="8">report_filename</field> + <field fid="9">rterid</field> + <field fid="10">submid</field> </fields> <records> <record> @@ -159,6 +189,8 @@ <value fid="6"><xsl:value-of select="loads/@load_average"/></value> <value fid="7"><xsl:value-of select="@version"/></value> <value fid="8"><xsl:value-of select="$report_filename"/></value> + <value fid="9"><xsl:value-of select="$rterid"/></value> + <value fid="10"><xsl:value-of select="$submid"/></value> </record> </records> </sqldata> @@ -175,7 +207,7 @@ <value fid="0"><xsl:value-of select="$rterid"/></value> <value fid="1" type="xmlblob"> <rteval_details> - <xsl:copy-of select="clocksource|network_config|loads|cyclictest/command_line"/> + <xsl:copy-of select="clocksource|services|kthreads|network_config|loads|cyclictest/command_line"/> </rteval_details> </value> </record> @@ -244,4 +276,14 @@ </records> </sqldata> </xsl:template> + + <xsl:template match="/rteval/cyclictest/system/histogram/bucket|/rteval/cyclictest/core/histogram/bucket" + mode="cyclic_histogram_sql"> + <record> + <value fid="0"><xsl:value-of select="$rterid"/></value> + <value fid="1"><xsl:value-of select="../../@id"/></value> + <value fid="2"><xsl:value-of select="@index"/></value> + <value fid="3"><xsl:value-of select="@value"/></value> + </record> + </xsl:template> </xsl:stylesheet> diff --git a/server/rteval_xmlrpc.py b/server/rteval_xmlrpc.py index 334ef8c..4362d89 100644 --- a/server/rteval_xmlrpc.py +++ b/server/rteval_xmlrpc.py @@ -35,11 +35,10 @@ from rteval.rtevalConfig import rtevalConfig def Dispatch(req, method, args): # Default configuration defcfg = {'xmlrpc_server': { 'datadir': '/var/lib/rteval', - 'xsltpath': '/usr/share/rteval', 'db_server': 'localhost', 'db_port': 5432, 'database': 'rteval', - 'db_username': 'xmlrpc', + 'db_username': 'rtevxmlrpc', 'db_password': 'rtevaldb' } } diff --git a/server/rtevaldb.py b/server/rtevaldb.py index e3c7928..d8453c9 100644 --- a/server/rtevaldb.py +++ b/server/rtevaldb.py @@ -27,71 +27,24 @@ import os from database import Database -from xmlparser import XMLSQLparser -def register_report(config, xmldata, filename, debug=False, noaction=False): +def register_submission(config, clientid, filename, debug=False, noaction=False): + "Registers a submission of a rteval report which signalises the rteval_parserd process" + dbc = Database(host=config.db_server, port=config.db_port, database=config.database, user=config.db_username, password=config.db_password, debug=debug, noaction=noaction) - parser = XMLSQLparser(os.path.join(config.xsltpath, "xmlparser.xsl"), xmldata) - - systems = parser.GetSQLdata('systems') - sysid = dbc.GetValue(systems, 0, 'sysid') - - # Check if system is already registered - chk = dbc.SELECT('systems',['syskey'], where={'sysid': sysid}) - if dbc.NumTuples(chk) == 0: - # This is a new system, register it - res = dbc.INSERT(systems) - if len(res) != 1: - dbc.ROLLBACK() - raise Exception, "** register_report(): Failed to register system [1]" - - syskey = res[0] - systemhost = parser.GetSQLdata('systems_hostname', syskey=syskey) - res = dbc.INSERT(systemhost) - if len(res) != 1: - dbc.ROLLBACK() - raise Exception, "** register_report(): Failed to register system hostname/ipaddr [1]" - - else: - # If this is a known system, check that hostname / IP address is the same - syskey = dbc.GetValue(chk, 0, 0) - systemhost = parser.GetSQLdata('systems_hostname', syskey=syskey) - srch = {'hostname': dbc.GetValue(systemhost, 0, 'hostname'), - 'ipaddr': dbc.GetValue(systemhost, 0, 'ipaddr')} - chk = dbc.SELECT('systems_hostname',['hostname','ipaddr'], where=srch) - if dbc.NumTuples(chk) == 0: - # This is an unknown hostname, register it - dbc.INSERT(systemhost) + submvars = {"table": "submissionqueue", + "fields": ["clientid", "filename"], + "records": [[clientid, filename]], + "returning": "submid" + } - # system is now registered, including hostname and IP address, and - # we have a reference in the 'syskey' variable. - - # Register rteval run - rterun = parser.GetSQLdata('rtevalruns', syskey=syskey, report_filename=filename) - res = dbc.INSERT(rterun) + res = dbc.INSERT(submvars) if len(res) != 1: - dbc.ROLLBACK() - raise Exception, "** register_report(): Failed to register rteval run [1]" - rterid = res[0] # RTEval Run ID - - # Register some more details about the run - rtedet = parser.GetSQLdata('rtevalruns_details', rterid=rterid) - dbc.INSERT(rtedet) - - # Register cyclic statistics data - cyclic = parser.GetSQLdata('cyclic_statistics', rterid=rterid) - dbc.INSERT(cyclic) + raise Exception("Could not register the submission") - # Register cyclic raw data - cycraw = parser.GetSQLdata('cyclic_rawdata', rterid=rterid) - dbc.INSERT(cycraw) - - # Commit this work dbc.COMMIT() - - # We're done - return (syskey, rterid) + return res[0] diff --git a/server/xmlparser.py b/server/xmlparser.py deleted file mode 100644 index 09afd64..0000000 --- a/server/xmlparser.py +++ /dev/null @@ -1,146 +0,0 @@ -# -# xmlparser.py -# Library for parsing rteval XML files -# -# Copyright 2009 David Sommerseth <davids@redhat.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For the avoidance of doubt the "preferred form" of this code is one which -# is in an open unpatent encumbered format. Where cryptographic key signing -# forms part of the process of creating an executable the information -# including keys needed to generate an equivalently functional executable -# are deemed to be part of the source code. -# - -import libxml2 -import libxslt -import hashlib -import StringIO -import types - -class XMLSQLparser(object): - "Class for parsing XML into SQL using an XSLT template for mapping data fields" - - def __init__(self, xslt, xml): - self.xml = self.__get_xml_data(xml) - - # Verify that this is a valid rteval XML file - try: - ver = float(self.xml.xpathEval('/rteval/@version')[0].content) - if ver < 0.8: - raise Exception, 'Unsupported rteval XML version' - except Exception, err: - raise Exception, 'Input file was unparsable or not a valid rteval XML file (%s)' % str(err) - - xsltdoc = self.__get_xml_data(xslt) - self.parser = libxslt.parseStylesheetDoc(xsltdoc) - - - def __get_xml_data(self, input): - if hasattr(input, '__module__') and (input.__module__ == 'libxml2') and hasattr(input, 'get_type'): - if input.get_type() == 'document_xml': - # It's an XML document, use it directly - return input - elif input.get_type() == 'element': - # It's an XML node, create a document and set node as root - xmldoc = libxml2.newDoc("1.0") - xmldoc.setRootElement(input) - return xmldoc - elif type(input) == types.StringType: - # It's a string, assume a file name - try: - return libxml2.parseFile(input) - except Exception, err: - raise Exception, "** ERROR ** XMLSQLparser::__get_xml_data('%s') failed to load file" % str(input) - - # If invalid input ... - raise AttributeError, "Unknown input type for XML/XSLT data (not a filename, xmlDoc or xmlNode)" - - - def __xmlNode2string(self, node): - doc = libxml2.newDoc('1.0') - doc.setRootElement(node) - - iobuf = StringIO.StringIO() - xmlbuf = libxml2.createOutputBuffer(iobuf, 'UTF-8') - doc.saveFileTo(xmlbuf, 'UTF-8') - retstr = iobuf.getvalue() - del doc - del xmlbuf - del iobuf - return retstr - - - def GetSQLdata(self, tbl, rterid=None, syskey=None, report_filename=None): - params = { 'table': '"%s"' % tbl, - 'rterid': rterid and '"%i"' % rterid, - 'syskey': syskey and '"%i"' % syskey, - 'report_filename': report_filename and '"%s"' % report_filename } - resdoc = self.parser.applyStylesheet(self.xml, params) - - # Extract fields, and make sure they are ordered/sorted by the fid attribute - fields = [] - tmp_fields = {} - for f in resdoc.xpathEval('/sqldata/fields/field'): - tmp_fields[int(f.prop('fid'))] = f.content - - for f in range(0, len(tmp_fields)): - fields.append(tmp_fields[f]) - - # Extract values, make sure they are in the same order as the field values - records = [] - for r in resdoc.xpathEval('/sqldata/records/record'): - rvs = {} - for v in r.xpathEval('value'): - if v.prop('type') == 'xmlblob': - fieldval = self.__xmlNode2string(v.children) - elif v.prop('isnull') == '1': - fieldval = None - else: - fieldval = v.content and v.content or None - - if v.hasProp('hash') and fieldval is not None: - try: - hash = getattr(hashlib, v.prop('hash')) - except AttributeError: - raise Exception, 'Unsuported hash algoritm: %s' % v.prop('hash') - - rvs[int(v.prop('fid'))] = hash(fieldval).hexdigest() - else: - rvs[int(v.prop('fid'))] = fieldval - - # Make sure the field values are in the correct order - vls = [] - for v in range(0, len(rvs)): - vls.append(rvs[v]) - - # Append all these field values as a record - records.append(vls) - - result = { 'table': resdoc.xpathEval('/sqldata/@table')[0].content, - 'fields': fields, 'records': records} - - # Extract the key field being returned from INSERT statements, if set - try: - retkey = resdoc.xpathEval('/sqldata/@key') - if retkey and retkey[0] and retkey[0].content: - result['returning'] = retkey[0].content - except: - pass - - resdoc.freeDoc() - return result - diff --git a/server/xmlrpc_API1.py b/server/xmlrpc_API1.py index 95390bd..7b61234 100644 --- a/server/xmlrpc_API1.py +++ b/server/xmlrpc_API1.py @@ -36,7 +36,7 @@ import rtevaldb class XMLRPC_API1(): def __init__(self, config=None, debug=False, nodbaction=False): # Some defaults - self.fnametrans = string.maketrans("/\\", "::") # replace path delimiters in filenames + self.fnametrans = string.maketrans("/\\.", "::_") # replace path delimiters in filenames self.debug = debug self.nodbaction = nodbaction self.config = config @@ -53,12 +53,12 @@ class XMLRPC_API1(): os.chdir(startdir) - def __getfilename(self, dir, fname, comp): + def __getfilename(self, dir, fname, ext, comp): idx = 0 if comp: - filename = "%s/%s/%s.bz2" % (self.config.datadir, dir, fname.translate(self.fnametrans)) + filename = "%s/%s/%s%s.bz2" % (self.config.datadir, dir, fname.translate(self.fnametrans), ext) else: - filename = "%s/%s/%s" % (self.config.datadir, dir, fname.translate(self.fnametrans)) + filename = "%s/%s/%s%s" % (self.config.datadir, dir, fname.translate(self.fnametrans), ext) while 1: if not os.path.exists(filename): @@ -90,14 +90,14 @@ class XMLRPC_API1(): # Save a copy of the report on the file system # Make sure we have a directory to write files into - self.__mkdatadir(self.config.datadir + '/reports/' + clientid) - fname = self.__getfilename('reports/' + clientid,'report.xml', False) + self.__mkdatadir(self.config.datadir + '/queue/') + fname = self.__getfilename('queue/', ('%s' % clientid), '.xml', False) xmldoc.saveFormatFileEnc(fname,'UTF-8',1) if self.debug: print "Copy of report: %s" % fname - # Register the report into a database and return the rteval run id - (syskey, rterid) = rtevaldb.register_report(self.config, xmldoc, fname, + # Register the submission and put it in a parse queue + rterid = rtevaldb.register_submission(self.config, clientid, fname, debug=self.debug, noaction=self.nodbaction) if self.nodbaction: rterid = 999999999 # Fake ID when no database registration is done @@ -116,7 +116,7 @@ class XMLRPC_API1(): self.__mkdatadir(self.datadir + '/uploads/' + clientid) # Get a unique filename, as close as possible to the input filename - fname = self.__getfilename('uploads/' + clientid, filename, not decompdata) + fname = self.__getfilename(('uploads/%s/%s' % clientid), filename, None, not decompdata) # Save and return filename used server side f = open(fname, "w") |