diff options
Diffstat (limited to 'doc')
111 files changed, 4503 insertions, 1379 deletions
diff --git a/doc/Makefile.am b/doc/Makefile.am index d38d4d0b..62ec7500 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -5,6 +5,7 @@ html_files = \ features.html \ generic_design.html \ expression.html \ + droppriv.html \ history.html \ how2help.html \ install.html \ @@ -32,6 +33,7 @@ html_files = \ dev_queue.html \ omsnmp.html \ ommysql.html \ + omoracle.html \ omlibdbi.html \ imfile.html \ imtcp.html \ @@ -76,6 +78,7 @@ html_files = \ rsconf1_filecreatemode.html \ rsconf1_filegroup.html \ rsconf1_fileowner.html \ + rsconf1_generateconfiggraph.html \ rsconf1_gssforwardservicename.html \ rsconf1_gsslistenservicename.html \ rsconf1_gssmode.html \ @@ -99,6 +102,31 @@ html_files = \ omrelp.html \ syslog_parsing.html \ troubleshoot.html \ + rsyslog_conf_actions.html \ + rsyslog_conf_examples.html \ + rsyslog_conf_filter.html \ + rsyslog_conf_global.html \ + rsyslog_conf_modules.html \ + rsyslog_conf_output.html \ + rsyslog_conf_templates.html \ + rsyslog_conf_nomatch.html \ + queues_analogy.html \ + multi_ruleset.html \ src/classes.dia -EXTRA_DIST = $(html_files) +grfx_files = \ + rsyslog_confgraph_complex.png\ + rsyslog_confgraph_std.png \ + direct_queue0.png \ + direct_queue1.png \ + direct_queue2.png \ + direct_queue3.png \ + direct_queue_rsyslog.png \ + direct_queue_rsyslog2.png \ + direct_queue_directq.png \ + dataflow.png \ + queue_analogy_tv.png \ + gssapi.png \ + rsyslog-vers.png + +EXTRA_DIST = $(html_files) $(grfx_files) diff --git a/doc/action-call.dot b/doc/action-call.dot new file mode 100644 index 00000000..86c6834d --- /dev/null +++ b/doc/action-call.dot @@ -0,0 +1,33 @@ +// This file is part of rsyslog. +// +// rsyslog action call state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot action-call.dot -Tpng >action-call.png + +digraph G { + label="\n\nrsyslog message states during action processing\nhttp://www.rsyslog.com"; + //fontsize=20; + + ok [label="ready for processing" color="green"]; + mpf [label="message permanent failure" color="red"]; + tf [label="temporary failure"] + cPen [label="commit pending"]; + com [label="committed" color="red"]; + + tf -> tf [label="retry fails, i < n"]; + tf -> mpf [label="retry fails, i = n"]; + tf -> ok [label="retry succeeds"]; + ok -> com [label="doAction RS_RET_OK"]; + ok -> cPen [label="doAction COMMIT_PENDING"]; + ok -> tf [label="doAction RS_RET_SUSPENDED"]; + ok -> mpf [label="doAction RS_RET_DISABLED"]; + cPen -> com [label="endTransaction RS_RET_OK"]; + cPen -> tf [label="endTransaction _SUSPENDED"]; + + //{rank=same; tf cPen} + {rank=same; com mpf} +} diff --git a/doc/action_state.dot b/doc/action_state.dot new file mode 100644 index 00000000..d56d9da0 --- /dev/null +++ b/doc/action_state.dot @@ -0,0 +1,33 @@ +// This file is part of rsyslog. +// +// rsyslog message state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + compound=true; nodesep=1.0 + //label="\n\nrsyslog action transaction states\nhttp://www.rsyslog.com"; + //fontsize=20; + + rdy [label="ready" group="main"]; + itx [label="in Tx" group="main"]; + comm [label="commit"] + rtry [label="retry"] + susp [label="suspended"] + + rdy -> itx [label="transaction begins"] + itx -> itx [label="success"] + itx -> comm [label="commit\n(caller or auto)"] + itx -> rtry [label="error"] + comm -> rdy [label="success"] + comm -> rtry [label="error"] + rtry -> rdy [label="recovered"] + rtry -> susp [label="could not\nrecover"] + susp -> rtry [label="timeout expired"] + + {rank=same; comm rtry} +} diff --git a/doc/batch_state.dot b/doc/batch_state.dot new file mode 100644 index 00000000..0dd48b47 --- /dev/null +++ b/doc/batch_state.dot @@ -0,0 +1,28 @@ +// This file is part of rsyslog. +// +// rsyslog batch state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + compound=true; nodesep=1.0 + //label="\n\nrsyslog batch states\nhttp://www.rsyslog.com"; + rankdir=LR + + rdy [label="ready"]; + bad [label="message-caused\nfailure"]; + sub [label="submitted"] + disc [label="discarded" color="red"] + + rdy -> sub [label="submitted to action"] + rdy -> bad [label="permanent fail"] + rdy -> disc [label="action requests discarding"] + sub -> rdy [label="next action or\naction-caused failure"] + bad -> rdy [label="next action"] + + //{rank=same; comm rtry } +} diff --git a/doc/dataflow.png b/doc/dataflow.png Binary files differnew file mode 100644 index 00000000..fd614d8c --- /dev/null +++ b/doc/dataflow.png diff --git a/doc/design.tex b/doc/design.tex new file mode 100644 index 00000000..53d25313 --- /dev/null +++ b/doc/design.tex @@ -0,0 +1,859 @@ +\documentclass[a4paper,10pt]{article} +\usepackage{amsmath} +\usepackage{amsfonts} +\usepackage{amssymb} +\usepackage{graphicx} +\usepackage{listings} +\usepackage{algorithm,algorithmic} +\usepackage{float} + +\pagestyle{headings} + +\newcommand{\IN}{\mathbb{N}} +\newcommand{\MM}{\mathcal{M}} +\newcommand{\QQ}{\mathcal{Q}} +\newcommand{\AAA}{\mathcal{A}} +\title{Rsyslog Design and Internals} +\author{Rainer Gerhards\\ +rgerhards@adiscon.com} + +\begin{document} + +\maketitle + +\begin{abstract} +This paper describes rsyslog design and internals. It is created to facilitate a discussion about the implementation of "batched queue processing". As such, it does not describe the full design of rsyslog but rather those elements that are relevant to queues. However, the document may be expanded in the future. +\end{abstract} + +\tableofcontents + +\section{Preliminaries} +\subsection{On the Use of English} +\begin{quotation} +\begin{flushright} +I ventured to write this book in English because ... \\ +it will be more easily read in poor English, \\ +than in good German by 90\% of my intended readers. \\ +--- HANS J. STETTER, Analysis of Discretization Methods for \\ +Ordinary Differential Equations (1973) +\end{flushright} +\end{quotation} + +There is not much I could add to Mr. Stetter's thought, except, maybe, that the number to quote probably tends more to 99\% in this case than to the 90\% Mr. Stetter notes. So please pardon those errors in language use that I have not yet been able to fix or even see. Suggestions for corrections and improvements are always welcome. +\subsection{Notational Conventions} +In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathcal{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in caligraphic letters ($\mathcal{O}$). Often, objects $O_i, i \in \IN, i \le |\mathcal{O}|$ partition $\mathcal{O}$, but this is not necessarily the case. + +\subsection{Definitions} +\subsubsection{Sudden Fatal Failure} +As sudden fatal failure is one that occurs at some instant and causes Complete loss of processing capabilities. The two major cases are a sudden power loss or a ``kill -9'' of the process. There are more exotic cases, too, like disasters. + +One may argue that it is possible to protect against many sudden fatal failure cases. For example, using an uninterruptable power supply (UPS) will prevent a sudden power loss. While this is true in most cases, it does not hold if looked very closely: in the case of the UPS, for example, a failure in the UPS itself may cause a sudden power loss, which can not be mitigated. Well, actually there can be several layers of mitigation, but always one more potential failure scenario remains. So it is not possible to totally solve the issue. + +The concept of ``sudden fatal failure'' now covers all these rest risk that result in termiantion of rsyslogd without the ability execute any code before this happens. This is a very important concept in regard to audit-gradeness. + +\subsubsection{Audit Grade} +In the context of this document, ``audit grade'' means that a subsystem never loses a message that it has taken responsibility for, not even in cases of sudden fatal failures. The only limit in this restriction is that a subsystem does not guarantee message survival if the subsytem at large is being destroyed (e.g. during a disaster) or some of its components are not of audit-grade. This draws a fine limitation on the audit-grade of a subsystem. + +For example, the rsyslog queue subsystem receives messages and acknowledges them to the submitter (e.g. an input), when they have been enqueued in the storage system. If the queue system is configured to provide audit-grade operation\footnote{Audit-grade queue operation is considerably slower than regular operations, as such this mode is not enabled by default. Most installations will never need a completely audit-grade queue}, the queue relies on the storage subsystem to work properly. If, for example, a disk read error occurs, the message may no longer be readable from the disk and as such is lost. The root cause here is that the disk subsystem was not of audit grade, because it otherwise would not have lost the message. So in this case the queue code is of audit grade, but the one of its components, the disk subsytem, was not. So the overall system is not of audit grade. + +To simplify talking about the audit-gradness of several subsytems, we assume that all of their subsystems are also of audit grade. In an actual deployment, however, this means the the system designer must carefully select audit-grade subsystems. Overlooking a single non-audit-grade component will make the whole system of not audit grade quality. + +Please note that it can be rather tricky to ensure a complete system is of audit grade. A border case is main memory integrity. Even with error-correcting memory, there may situations arise where a memory error occurs (probably due to a very unlikely series of well-hitting cosmic rays) that is unrecoverable. At this point, system integrity is at risk. The only real solution is to immediately shut down the system and restart it (without giving any process a chance to execute). Note, however, that in an extreme view, an operating system routine that does so can also be considered dangerous, as memory in use by this routine might be affected by the malfunction. We could extend this scenario and further complicate it, but that goes beyond the scope of this paper. The example was primarily meant to show how subtle audit-grade reliability is. + +In rsyslog, we currently use a slightly \marginpar{duplication\\permitted}relaxed consistency condition for message integrity inside an audit-grade subsystem. While we do not accept message loss, we permit slight message \emph{duplication}, but only in exceptional cases. This is permitted because, with proper message generation, the dulication problem can be easily fixed at the end-to-end layer. For example, the original sender can include a UUID, which can be used to sort out duplicates at the final destination. Insisting on not allowing duplication complicates matters and is often impossible with today's logging protocols. So, for the time being, we aim at this relaxed criteria, which is hard enough to achive. After we have achieved that goal, we may further try to solve the duplicaton problem. Some hooks already exist. But we do not guarantee such an effort will be made any time soon. + +\section{Overall Design} +From a high-level prespective, rsyslogd is ``just'' a high-performance message router. It accepts messages from various sources, applies user-configured filters to them, and routes potentially transformed messages to destinations based on these filters. +\section{Objects} +\subsection{Plugins} +Plugins provide code potentially written by a third party to extend rsyslog. + +Conceptually, a plugin is a tuple of callable functions $(\phi_1, \phi_2, \ldots)$ which implement an interface. There are three different types of plugins: input, output and library. The plugin type denotes the primary interface implemented by the plugin. Additional interfaces may be implemented\footnote{This is not yet done in plugins, but is possible and assumed to be done at a later point in time}. + +In the context of this paper, the output plugin interface is most important. It implements three entry points: + +\paragraph{doAction()} +is used to submit messages to the output plugin. The entry point may or may not commit the messages to their ultimate destination. + +\paragraph{beginTransaction()} +is used to inform the plugin that a new transaction begins. It must prepare for processing. + +\paragraph{endTransaction()} +is indicated that the upper layer \emph{needs} to close the transaction. If there is any uncommited data left, it must be commited or rolled back. + +Every instance of an output plugin is guaranteed \emph{not} to be called concurrently by multiple threads. Further, no context switch will happen between calls to $doAction()$ and $endTransaction()$. + +\subsection{State Sets} +Several object have associated state based on a specific state set. These state sets are described together with the objects. + +As a general rule, individual state is associated with all instances $o$ of a class of objects. This state is called the object's \marginpar{state component} \emph{state component} $s$. If we want to obtain an object's state, we write $S(o)$. Please note that $S(o)$ is only defined for those objects that have a state component. + +\subsection{Messages} +A message $m$ represents a a single syslog message inside the system. It is a tuple of attributes. Some of these attributes directly orginate from the message content, some others are meta-information taken from the context. For example, there is an meta-attribute ``time of reception'' which conveys when the message was received by rsyslog's input subsystem. We do not list attributes here, as there are many and it is not of importance which exactly they are. + +The set $\MM$ is composed of all messages that exist at a given time inside rsyslog. + +\subsection{Queue} +A queue +$$Q = (C, \Phi, M)$$ +is a triplet of a set of configuration parameters $C$, a set of callbacks $\Phi$ and a set of messages $M \subseteq \MM$. + +If we need to obtain the set of message from a queue, we write $M(Q)$. The elements of the set of configuration parameters are written as $C_{param}$ where $param$ is an abbreviation of the parameter's meaning. To obtain a specific parameter from a queue, we write $C_{param}(Q)$. The most important elements of $C$ are: + +\paragraph{$C_{type}$} which denotes the queue implementation type. Most importantly, this selects from a set of queue drivers (for example disk-only or in-memory driver), which affects the basic operation of the queue instance. + +\paragraph{$C_{mMsg}$} which denotes the upper bound on the cardinality of $M$. + +\paragraph{$C_{mBatch}$} which denotes the upper bound of the cardinality of message batches created for this queue. + +Be $\QQ = \{Q_m, Q_1, Q_2, \ldots, Q_{|\AAA|}\}$ the set of all queues that exist inside rsyslog after the configuration file has been processed, with $|\QQ| = |\AAA| + 1$. + +Then +$$M_0 = \MM \setminus \bigcup_{i=1}^{|\QQ|} Q_i(M)$$ +\marginpar{at-risk-set}is the set of non-queued messages. The messages have either never been enqueued or have been dequeued but not finally been processed. This set represents the messages that may potentially be lost during an unclean shutdown of rsyslogd. This is why I call this set the ``\emph{at-risk-set}''. + + +\subsection{Batches} +A batch represents multiple processable messages. It is a unit of processing inside rsyslog's output system. Batches are used to dequeue a number of messages from a queue and then submit them to the lower action layer. Batches are natural \emph{transaction boundaries}, in the sense that multiple output transactions may be done on the messages inside a batch, but each transaction must end at the end of the batch. A batch is always associated to a specific queue $Q$. + +A batch +$$B = (b_1, b_2, \ldots, b_n )$$ +is a $n$-tuple of \marginpar{processable\\message}processable messages +$$b = (m, s)$$ +which are an ordered pair of a message $m$ and an associated processing state $s$. To denote the $n$-th message inside the batch, we write $m(b_n)$, to denote the status component of the $n$-th message, we write $S(b_n)$. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{batch_state.jpeg} +\end{center} +\caption{batch message processing states} +\label{fig_batchmsg_states} +\end{figure} + +The state set for the processing states is defined as follows: +$$ +S_B = \{ rdy, bad, sub, disc \} +$$ + +With the semantics of the various states being the following: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Semantics \\\hline + rdy & ready for processing\\ + bad & this message triggered an unrecoverable failure in action\\ + & processing and must not be resubmitted to this action\\ + sub & message submitted for processsing, result yet unknown \\ + disc & action sucessfully processed, but must not be submitted \\ + & to any further action in action unit \\\hline +\end{tabular} +\end{center} +The associated state diagram is shown in figure \ref{fig_batchmsg_states} on page \pageref{fig_batchmsg_states}. + +Batch sizes vary. The actual cardinality is a function of the cardinality of $M(Q)$ at the time of batch creation and the queue configuration: + +$$1 \leq |B| \leq \max(C_{mBatch}(Q), |M(Q)|)$$ + +\subsection{Action Unit} +An action unit +$$u = (f, a_1, \ldots, a_n), a_i \in \AAA \text{ for } i \in \IN, i \le n$$ +is a tuple consisting of a filter function $f$ and $n \in \IN$ actions. \emph{Does rsyslog still support nonsense action units with $n=0$? - check!} + +\subsection{Action} +An action +$$a = (a_C, a_\psi)$$ +is an ordered pair of a tuple of configuration attributes $a_C$, and a tuple of processing functions $a_\psi$. Be the set $\AAA$ composed of all actions that exist in rsyslog after the configuration file has been processed. + + +\section{Processing} +\subsection{Object States} +Various objects keep state. Some of these objects, like messages, batches and actions seem to share state. However, thinking about shared state leads to very complex setup. As such, state is modelled for each object $o$ individually. Instead, the state function $S_O(o)$ can be used to obtain an obtain an individual objects state. That state can be used to modify the state diagrams of the other objects with which relationships exist. + +\subsubsection{Actions} +Actions are provided by output plugins. An action enables the engine to write messages to some destination. It is important to note that ``destination'' is a very broad abstraction. A destination may be a file inside a local or remote file system, a database table or a remote syslog server in another network. + +Actions are transactional in the following sense: more than one message can be submitted to an action. The action does not necessarily process the submitted messages unless the caller ends the transaction. However, the action itself may also end the transaction and notify the caller. This is \emph{not} considered an error condition and \emph{must} be handled gracefully by the caller. If a transaction aborts, the caller \emph{must} assume that none of the elements submitted since the begin of transaction have been processed. The action will try to backout anything that was already processed at the time the transaction failed. However, not all outputs work on actually transactional destination. As such, an action is permitted not to backout incomplete interim results. As such, after a transaction abort, some message duplication may occur. We call this the \emph{relaxed integrity condition} for actions. + +An output transaction is started by calling \emph{beginTransaction()} either explicitely or implicitely by a call to \emph{doAction()} without calling \emph{beginTransaction()} before. Then, one or more calls to \emph{doAction()} follow. When the caller intends to finish the transaction, it calls \emph{endTransaction()}. However, the transaction may also be terminated from the action itself in response to a \emph{doAction()} call. + +Mathematically, an action transaction builds a totally ordered set of uncommitted messages $M_u$. The order relation is defined over the sequence in which messages are being provided to \emph{doAction()}. At any time a commit is attempted, the full set $M_u$ is committed and may either succeeed completely or not at all (in the sense of the relaxed integrity condition described above). + +A commit is attempted when +\begin{enumerate} +\item the caller decides to call \emph{endTransaction()} +\item or earlier if the action decides it needs to commit now (e.g. because of buffers filling up). +\end{enumerate} + +In the seconds case, the action may decide to commit all message but the current one or all (this is depending on action logic). So if the action decideds to commit a transaction before the caller calls \emph{endTransaction()}, a set of commited messages $M_c$ is build and $M_u$ is modified. Be $n$ the $n$-th iterated \emph{doAction()} call and $m_n$ the current message of this call, then the sets are build as follows: + +\begin{algorithm} +%\caption{} +\begin{algorithmic} +\IF{action commits $m_n$} + \STATE $M_c = M_u \cup m_n$ + \STATE $M_u = \emptyset$ +\ELSE + \STATE $M_c = M_u$ + \STATE $M_u = \{ m_n\}$ +\ENDIF +\end{algorithmic} +\end{algorithm} + +In other words, if anything is committed early, it is always the full set $M_u$, with or without the current message. The caller needs to know which messages are already commited. As \emph{doAction()} finishes one transaction and starts a new one in a single call, we can not use action state the let the caller know this happened. So we use our above finding and just convey back if the transacton is still continuing or the current message or all others before it were committed. The caller must then act accordingly. Please note that when an error happens, the whole transaction must still be considered failed. As such, ``partial commit'' states need not to be mixed with failure states. + +Please note that the above method leaves a small potential issue unaddressed: if the action does an early commit of $M_u \setminus m_n$, an error happens when adding $m_n$ to the new $M_u$ (like running out of resources), the action would need to convey both the successful transaction as well as the failure state. This is not possible with the current interface. We could use callbacks to provide such notification, but this complicates the code. So, if that situaton arises, the action must temporarily buffer the error condition and convey it as part of either the next \emph{doAction()} call or during \emph{endTransation()} processing. This can be done, for example, by advancing its internal state accordingly. + +The state set for a actions is defined as follows: +$$ +S_A = \{ rdy, itx, comm, rtry, susp, died \} +$$ + +With the semantics of the various states being the following: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Semantics \\\hline + rdy & ready, waiting for transaction begin\\ + itx & in transaction, accept more data \\ + comm & transaction finished \\ + rtry & action failed but may be able to recover \\ + susp & action currently defunctional until timeout expires \\ + died & unrecoverable error condition occured, no longer usable \\\hline +\end{tabular} +\end{center} + +In the associated state diagram in figure \ref{fig_action_states}, we do not include the \emph{died} state, because it is entered whenever a totally unrecoverable error state may occur. This is a very exceptional incident (which most output plugins do not even support), so we have kept the diagram simple. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.5]{action_state.jpeg} +\end{center} +\caption{Action State Diagram} +\label{fig_action_states} +\end{figure} + +\emph{Note well} that the state diagram describes the action state. It does \emph{not} describe the transaction state. While action- and transaction state are closely related to each other, they are different entities. + +The return code of \emph{doAction()} and \emph{endTransaction()} is used to convey the transaction state. As such, it is a function of the actions's current state after processing the request. The mapping is as shown below: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Return Code (RS\_RET\_\ldots)\\\hline + rdy & OK \\ + itx & COMMITTED (if there was an auto-commit without $m_n$)\\ + & DEFER\_COMMIT (if there was no auto-commit)\\ + comm & internal state, not to be exposed to upper layer \\ + rtry & SUSPENDED \emph{(new code needed)} \\ + susp & SUSPENDED \\ + died & DISABLED \\\hline +\end{tabular} +\end{center} + +For the rest of this document, let's assume there is a function \emph{getReturnCode()} that implements this mapping. + +It is important to think about how retries are handled. There is a user-configured per-action upper number of retries $C_r$ and retry interval $C_i$. In \emph{rsyslog v3}, there is no concept of output transactions. As such, only single messages are processed. When a temporary action failure occurs, the action is re-tried $C_r$ times, where the action processing thread is waiting in a \emph{sleep()} $C_i$ operating system API call\footnote{a suitable API is used, not \emph{sleep()} itself}. If the action succeeds during the retry processing, everything continues as usual. If it does not succeed, two things happen: +\begin{itemize} +\item the message is flagged as ``action permanent failure'' (what may trigger backup processing) +\item the action is actually suspended for $C_i$ seconds +\end{itemize} +If then a new message is sent to the action, and $C_i$ seconds have not yet elapsed, the action is flagged as having failed without being re-tried again\footnote{During the analysis for this paper, it was seen that actually $C_r$ retries are attempted in v3, but each of them will never actually re-try the action. This is a software bug, which does not cause any harm and thus will not be fixed in v3. The new implementation in v4 will obviously not inherit this problem}. This is done in an effort to reduce resource utilization and prevent the system from slowing down e.g. by too-many retries to a remote server that went offline. + +With transactional output mode in \emph{rsyslog v4}, the logic above can no longer work. First of all, retrying single actions does not help, because all of the current transaction needs to be resubmitted. As such, the upper layers need to be notified of failure. Then, they need to resubmit the batch. In that design, the lower layer needs to return immediately after detecting the failure. Recovery handling is now to be done when the next transaction is started. However, we must make sure that we do not do excessive retries. So retry processing is only to be carried out if it was not tried less than $C_i$ seconds ago. + +The required functionality can be implemeted by a \emph{prepareAction} function that readies the action for processing if there is need to do so. That function is then called in all entry points before anything else is done. Then, actual processing is carried out and the resulting action state be used to generate the return code for the upper-layer caller. Find below a rough pseudocode to do so: + +\lstset{language=python} +\begin{lstlisting} +def prepareAction(): + if state == rtry: + try recovery (adjust state accordingly) + if state == rdy: + beginTransaction() [output plugin] + +def processMessage(message): + prepareAction() + if state == itx + doAction(message) [output plugin] + return getReturnCode() + +def doEndTransaction(): + prepareAction() + if state == itx + endTransaction(); [output plugin] + return getReturnCode() +\end{lstlisting} + +\subsection{Output Subsystem Layers} +The rsyslog engine is organized in layers, where each layer is represented by the dominating object: + +\begin{figure} +\includegraphics[scale=0.75]{rsyslog_output_layers.jpeg} +\label{rsyslog output layers} +\end{figure} + +If looking at the data flow, a queue dequeues batches of messages, which are than run through a generic action system and put into output plugins. Note that on the batch layer, only batches are supported as units of work, whereas the action layer is message-oriented but supports transactions of multiple messages. This is done by indicating when a transaction necessarily needs to end (that point being the end of batch from the batch layer). + +The plugins can be written by third parties and are roughly comparable to minidrivers. The generic action system provides all complexity of action processing wheras the output plugin provides a limited set of callbacks that enable the generic framework to talk to the actual destination system. As such, writing outputs is a very simple task. However, rsyslog does not limit the creation of very complex outputs, which may be able to offer superior performance for some destinations. + +\subsection{Output Failure} +\subsubsection{Cases} +When an output action is called, it may encounter a failure condition. In general, there are two different cases: +\begin{enumerate} +\item action caused failures +\item message-content caused failures +\end{enumerate}. + +Failures rooted in the action are things like broken network connections, file systems run out of space or database servers that are down. Most importantly, the failure is not related to message content. As such, it is appropriate to retry the action with the same message until it finally succeeds (assuming that someone restores the system in question to proper operation). We can not expect that the problem is cleared just by discarding the current message and re-trying with the next one. + +In my view, action caused failures are the far majority of all failures. For rsyslog versions 3 and below, all rsyslog-provided plugins consider failures to be action-caused and thus potentially recoverable by simple retry. With the only exception being fatal error conditions that render the whole action unusable. + +David Lang pointed out, that there may also exist error conditions that are not caused by the action (or the subsystem it talks to) itself, but rather by message data. He provided the following samples where message content can cause permanent issues with action execution: + +\begin{itemize} +\item unicode text causing grief +\item dynafile hits a read-only file +\item basicly data-driven things that trigger bugs in the message delivery +mechanism in some form. +\end{itemize} + +As David Lang said ``In an ideal world these would never happen, but for most output types I can think of some form of corrupt input that could cause that message to fail.''. +So this class of failure conditions actually exists. No matter how often the action retry mechanism is called, it will never succeeds (one may argue that the read-only dynafile is fixable, but we could replace that sample with an invalidly generated filename). The proper cure for these actions is to find the offending one and discard it. + +In conclusion, actions need to return different error states for these two different types of failures. Traditionally, RS\_RET\_SUSPENDED is returned when an action specific failure is hit. Most existing plugins also do this if a message-related failure occured, simply because they did not yet know that this situation exists. However, plugins also return different error codes, and at least these can be treated to mean message-permanent failures. To support this, a change to plugins is still required, because many simple return SUSPENDED state if anything went wrong (replacing the real error condition with SUSPENDED). A dedicated PROBABLE\_INVALID\_MSG return state is probably useful so that an output plugin can convey back that it consideres the message to be bad. On the other hand, this implies that the plugin must try to detect those, what means that the developer must think about all potential message-causes problems. That approach can be considered unreliable and as such it may be better not to provide such a dedicted state. + +\subsubsection{Handling of Failures} +In spite of the two different failure cases, different handling is needed for them. The action-based failure cases can and must be handled on the action level. As transactions abort when a failure occurs, support from the upper ``batch layer'' is necessary in order to handle resending batches of messages. + +For message-caused failure cases, the offending message must be found and then be discarded. A complexity here is that while a failure-causing message is being searched for, an action-based failure might occur. In that case, first the action-based failure condition must be solved, before the search for the problem message can continue. + +One approach might be that when the action-layer conveys back an action-caused failure (SUSPENDED), the batch layer knows that it simply needs to restart the full transaction (but not start an ``invalid message search''). If a message-based error condition is conveyed back, the batch system can not restart the full batch. Instead, it needs to enter search mode, where it creates partitions of the original batch, and calls itself recursively (at least in theory) on each of the subsets. + +Then, the same handling applies until either a failing message has been found or all messages have been successfully processed. Note that in the recursive step, action-based failures are recovered by full batch resubmits. This solves the above-mentioned complexity in a consistent way. + +If a binary-search-like method is used to detect failing records\footnote{This was originally suggested by David Lang.}, recursion may not really be an issue, as the recursion depth is limited to $\log_2 |B|$ where $B$ is the message batch. + +A message-caused failure can be rooted in one or more messages. One important question is if it is expected that the failure is caused by a single or multiple messages. Both is possible, so it is a question of probability. If we assume that it is more probable that a single messages causes the problems, it is useful to immediately return back to full batch submission of transactions once a problem-causing message has been identified. But then, if there are multiple problem-causing messages inside the batch, we may need many more iterations. + +If, on the other hand, we assume that it is more probable that multiple messages cause problems, it may make sense to keep resubmitting only subsets of the batch. However, then the performance is suboptimal if actually only one message was problematic. A solution might be to pick a compromise, e.g. first assume that a single message is problematic, but assume the opposite as soon as a second message with problems has been found. + +A potential algorithm for processing $n \le |B|$ messages from batch $B$ is described below. In the pseudocode, a ``processable'' message is one that neither is already committed nor had a permanent failure with this action. The term ``mpf'' means ``message permanent failure'' for this action (this will later be described in a batch state set). + +\begin{small} +\lstset{language=python} +\begin{lstlisting} +def submitBatch(B, n): + foreach processable message in + (first [at most] n messages of batch): + call processMessage + if action-caused failure: + retry full batch + if action-caused permanent failure: + mark all n messages as mpf + return + if auto-commit: + mark commited messages in batch as committed + if message-caused failure: + if n == 1: + mark message as mpf + return + else: + call submitBatch(B, n/2) + call submitBatch(B, n/2) +\end{lstlisting} +\end{small} + +After submitBatch() has completed, all messages are either committed or in mpf state. + +Note that an action-caused permanent failure occurs if an action-caused failure can not be resolved with the operator-configured number of retries. It will never occur if the user configured infinite retries. While an action is suspended, all calls will result in an action-caused permanent failure. Please keep in mind that these will be resubmitted to any backup actions inside the action unit, so the action's ability to cause permanent failure states is vital for a number of use cases (backup syslog server, to name just one). + +Batch processing inside an action unit thus can follow these strucuture: + +\begin{algorithm} +\caption{processBatch(B)} +\begin{algorithmic} +\FORALL{action $a$ in action unit} + \IF{execute action only on messages that failed before} + \STATE $n = |\text{messages in batch in mpf state}|$ + \STATE change mpf state back to ready + \ELSE + \STATE $n = |B \setminus \text{msgs with state discard}|$ + \STATE change all message states $\ne$ discard to ready + \ENDIF + \IF{$n >0$ } + \STATE call submitBatch(B, n) for action $a$ + \ENDIF +\ENDFOR +\end{algorithmic} +\end{algorithm} + +\paragraph{Why is it Important to differentiate the failure cases?} +This text originates from the mailing list and must be merged in. I provide it in the form it is, so it will not be forgotten (plus, it conveys the information). + +One may think that it is not necessary to differentiate between action-caused and message-caused failures. However, not doing so introduces subtle issues, because +then you either + +A) do not need the batch logic at all (because the action is configured for +infinite retries) + +Or + +B) you loose many messages if the action is not configured for infinite +retries and you have a longer-duration outage e.g. on a database server. +Let's say it is offline for a couple of hours, then you lose almost +everything in that period + +To prevent this, you need two different retry methods. + +One may argue that it is hard to differentiate between the two failure cases. This is correct. Buit I think it mostly depends on the quality of the output module. + +First of all, ``mostly'' implies that there may be some other cases, where it +really is impossible to differentiate between the two. In that case, I would +treat the issue as an action-caused failure. There are two reasons for this: + +1) rsyslog v3 currently does this always and not even a single person +complained about that so far. This is an empiric argument, and it does not +mean it caused problems. But it carries the co-notation that this seems not +to be too bad. + +2) If we would treat it as message-caused failure, we would no longer be able +to handle extended outages of destination systems, which I consider a vitally +important feature. + +When weighing the two, I know of lots of people who rely on 2), in sharp +contrast to knowig noone having problems with 1). So my conclusion is that it is +less problematic to define an otherwise undefinable failure reason to be +action-caused. Even more so as I assume this problem only exists in the +minority of cases. + +Now back to the quality of the output module: thinking about databases, their +API is usually very good at conveying back if there was a SQL error or a +connection abort. So while a SQL error may also be an indication of a +configuration problem, I would strongly tend to treat it is a being +message-caused. This is under the assumption that any reasonable responsive +admin will hopefully test his configuration at least once before turning it +into production. And config SQL errors should manifest immediately, so I +expect these to be fixed before a configuration runs in production. So it is +the duty of the output module to interpret the return code it received from +the API call and decide whether the failure is more likely action-caused or +message-caused. For database outputs, I would assume that it is always easy +to classify failures that must be action-caused, especially in the +dominating cases of failed network connections or failed servers. + +For other outputs it may not be as easy. But, for example, all stream network +outputs can detect a broken connection, so this also is a sure fit. + +For dynafiles, it really depends on how hard the output module is tries to differentiate +between the two failure cases. But I think you can go great length here, too. +Especially if you do not only look at the create() return code, but, iff a +failure occurs, you do more API calls to find out the cause. + +So I think the remaining problem is small enough to cause not too much issues +(and if so, they are unavoidable in any case). In conclusion, the two failure states are not only necessary, but can sufficiently sure enough be detected. + +\subsection{Random Topics} +I have begun to gather material from the mailing list in this section, because I feel it may be useful for others as well. Right now, the information is well hidden in the mailing list archives and there may be value in combining it all in one place. + +Due to the nature of this material, there is no specific organization between the subchapters and also formatting and language doesn't deny its rooting in the mailing list. + +\subsection{Reliability of Message Dequeueing} +A batch is actually dequeued when it is taken off a queue. So if at that point we +have a system power failure (for whatever reason), the messages are lost. +While the rsyslog engine intends to be very reliable, it is not a complete +transactional system. A slight risk remains. For this, you need to understand +what happens when the batch is processed. I assume that we have no sudden, +untrappable process termination. Then, if a batch cannot be processed, it is +returned back to the top of queue. This is not yet implemented, but is how +single messages (which you can think of an abstraction of a batch in the +current code) are handled. If, for example, the engine shuts down, but an +action takes longer than the configured shutdown timeout, the action is +cancelled and the queue engine reclaims the unprocessed messages. They go +into a special area inside the .qi file and are placed on top of the queue +once the engine restarts. + +The only case where this not work is sudden process termination. I see two +cases: + +a) a fatal software bug +We cannot really address this. Even if the messages were remaining in the +queue until finally processed, a software bug (maybe an invalid pointer) may +affect the queue structures at large, possibly even at the risk of total loss +of all data inside that queue. So this is an inevitable risk. + +b) sudden power fail +... which can and should be mitigated at another level + +One may argue that there also is + +c) admin error +e.g, kill -9 rsyslogd +Here a fully transactional queue will probably help. + +However, I do not think that the risk involved justifies a far more complex +fully transactional implementation of the queue object. Some risk always +remains (what in the disaster case, even with a fully transactional queue?). + +And it is so complex to let the messages stay in queue because it is complex +to work with such messages and disk queues. It would also cost a lot of +performance, especially when done reliably (need to sync). We would then need +to touch each element at least four times, twice as much as currently. Also, +the hybrid disk/memory queues become very, very complex. There are more +complexities around this, I just wanted to tell the most obvious. + +So, all in all, the idea is that messages are dequeued, processed and put +back to the queue (think: ungetc()) when something goes wrong. Reasonable +(but not more) effort is made to prevent message loss while the messages are +in unprocessed state outside of the queue. + +\paragraph{More reliable can actually be less reliable} +On the rsyslog mailing list, we had a discussion about how reliable rsyslog should be. It circles about a small potential window of message loss in the case of sudden fatal failure. Rsyslog can be configured to put all messages into a disk queue (instead of main memory), so these messages survive such a powerfail condition. However, messages dequeued and scheduled for processing during the power outage may be lost. + +I now consider a case where we have bursty UDP traffic and rsyslog is configured to use a disk-only queue (which obviously is much slower than an in-memory queue). Looking at processing speeds, the max burst rate is limited by using an ultra-reliable queue. To avoid using UDP messages, a second instance could be run that uses an in-memory queue and forwards received messages to the one in ultra-reliable mode (that is with the disk-only queue). So that second instance queues in memory until the (slower) reliable rsyslogd can now accept the message and put it into the reliable queue. Let's say that you have a burst of $r$ messages and that from these burst only $r/2$ can be enqueued (because the ultra reliable queue is so slow). So you lose $r/2$ messages. + +Now consider the case that you run rsyslog with just a reliable queue, one that is kept in memory but not able to cover the power failure scenario. Obviously, all messages in that queue are lost when power fails (or almost all to be precise). However, that system has a much broader bandwidth. So with it, there would never have been r messages inside the queue, because that system has a much higher sustained message rate (and thus the burst causes much less of trouble). Let's say the system is just twice as fast in this setup (I guess it usually would be *much* faster). Than, it would be able to process all r records. + +In that scenario, the ultra-reliable system loses $r/2$ messages, whereas the somewhat more "unreliable" system loses none - by virtue of being able to process messages as they arrive. + +Now extend that picture to messages residing inside the OS buffers or even those that are still queued in their sources because a stream transport blocked sending them. + +I know that each detail of this picture can be argued at length about. + +However, my opinion is that there is no "ultra-reliable" system in life, only various probabilities in losing messages. These probabilities often depend on each other, what makes calculating them very hard to impossible. Still, the probability of message loss in the system at large is just the product of the probabilities in each of its components. And reliability is just the inverse of that probability. + +This is where *I* conclude that it can make sense to permit a system to lose some messages under certain circumstances, if that influences the overall probability calculation towards the desired end result. In that sense, I tend to think that a fast, memory-queuing rsyslogd instance can be much more reliable compared to one that is configured as being ultra-reliable, where the rest of the system at large is badly influenced by this (the scenario above). + +However, I also know that for regulatory requirements, you often seem to need to prove that a system may not lose messages once it has received them, even at the cost of an overall increased probability of message loss. + +My view of reliability is much the same as my view of security: there is no such thing as "being totally secure", you can just reduce the probability that something bad happens. The worst thing in security is someone who thinks he is "totally secure" and as such is no longer actively looking at potential issues. + +The same I see for reliability. There is no thing like "being totally reliable" and it is a really bad idea to think you could ever be. Knowing this, one may begin to think about how to decrease the overall probability of message loss AND think about what rate is acceptable (and what to do with these cases, e.g. "how can they hurt"). + +\paragraph{Different Use Cases} +As David Lang pointed out, there exist different use cases for different levels of reliability. Most importantly, there exist use cases that do not demand very high throughput but rather ultra-realiability of the queue system. Here, ultra-reliability is just another word for the queue being of ``audit-grade''. Even if the queue provides audit-grade, the overall system is only then of audit-grade when all other components - most notably the transport protocols spoken by the inputs and outputs - are also of audit-grade. Most importantly, this means that an audit-grade system purely based on the IETF syslog protocol series can not be build. + +Used together with truly reliable protocols \emph{and} senders that block processing until a final acknowledgement has been received, an audit-grade system can potentially build based on rsyslog. To do so, an audit-grade queue subsystem is required, which is not present in releases less than 4.1.? (most importantly, v2 and v3 do not provide this capability). + +\subsection{Audit-Grade Queue Operations} +\subsubsection{Perquisites} +Audit-grade queue operations certain perquisites: +\begin{itemize} +\item rsyslog engine is of version 4.1.? or greater +\item disk-only queue type +\item checkpoint interval set to 1 +\item queue is configured to not permit losing any messages\footnote{The queue has several settings that can be used to fine-tune situations in which it may discard messages intentionally. All of these must be turned off. Most importantly, that means the producer is blocked for an infinite time if the queue is full.} +\item queue consumer must also be of audit-grade +\end{itemize} +Only when these prequisites are met, queue operation can be considered of being audit-grade. Note that when message loss in case of sudden fatal failure and similar incidents is acceptable, neither disk-only queues nore a checkpoint interval of 1 is necessary. Such a configuration can also be build with rsyslog v3, which is up to that level. + +Note that in the sections below we describe the implementation in broader terms. Most importantly, we do not restrict ourselves to disk-only queue storage drivers. This is important, because it simplifies design and opens the capability to introduce new, possibly faster-performing, queue storage drivers in the future. + +But it is important to keep in mind that a concrete queue is only of audit-grade if it matches all the perquisites given here, most importantly with the right configuration. + +\subsubsection{Implementation Alternatives} +Messages, or more precisely objects\footnote{While rsyslog deals with messages, the queue is designed to handle any type of thing that is represented as an rsyslog object. This is considered useful as queues may at some time contain other things than just messages, so we keep it generic.}, are enqueued by the queue producer (either an input module or the main message queue's consumer). The enqueue operation is completed only when the message has been successfully accepted by the queue storage driver. Then and only then the producer is permitted to remove the object from its own storage system. A rough sketch is given in algorithm \ref{alg_q_enq}. + +\begin{algorithm} +\caption{enqueueObject($o$)} +\begin{algorithmic} +\label{alg_q_enq} +\STATE lock queue mutex +\WHILE{queue is not ready for enqueue} + \STATE wait on queue to become ready +\ENDWHILE +\STATE call queue store driver to add $o$ +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +The dequeue-operation is more complex. We must ensure that each object stays in the queue until it is finally processed. Hereby, an object is finally processed, when processing of it has been completed. Remember that to enhance performance, objects are dequeued in batches of many. So at any given time, multiple messages may be processed, but not necessarily have finally completed doing so. If another worker thread then tries to obtain a new batch for processing, those ``in-process'' message must not be handed out a second time. Also, if a sudden fatal failure occurs during processing, queue operation must restart at the point of last commit. This means that all ``in-process'' messages need to be changed back to ``no processed'' state and be restarted again. In those cases the (acceptable) slight message duplication can occur. + +In our design, we differentiate between ``logical'' and ``physical'' dequeuing of batches. If a batch is generated for processing, it is logically dequeued --- in the sense that no other batch generating request will be able to receive another copy of these messages. If no exceptional situation happens, those messages will be processed and thus can be considered consumed under normal circumstances. + +However, actual deletion from the physical queue storage happens only after the batch is fully processed. At this point, all objects have been acknowledged by their destinations, which now have the responsibility for the object's survival. Consequently, we can delete them from the queue store. This process is considered the ``physical'' dequeue of the object. + +In order to find some simpler terms, we will call the logical dequeue operation just ``dequeue'' and the physical dequeue operation ``delete''. This is consistent with all previous work on rsyslog and thus probably leads to the least surprise when reading older source code and documentation. + +A first idea for a deletion is given in algorithm \ref{alg_pdeq_batch_1} (remember that $O(b)$ contains all objects within the given batch $b$, this is \emph{not} $O$-notation and should probably in the future be replaced by something else). + +\begin{algorithm} +\caption{deleteBatch($b$), first approach} +\begin{algorithmic} +\label{alg_pdeq_batch_1} +\STATE lock queue mutex +\FORALL{$o \in O(b)$} + \STATE find $o$ in queue storage + \STATE remove $o$ and keep queue structures intact +\ENDFOR +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +This algorithm is simple, but requires searching the queue store for the object to be deleted -- a potentially lengthy operation. However, we can improve the searching process if we know more about the inner structure of batch objects. It seems appropriate to dequeue objects in queue-sequential order. A drawback of doing so is that we must prevent other worker threads from trying to dequeue concurrently. This is not really a drawback. We need to guard dequeue operations by a mutex in any case, because otherwise internal structures can not be kept consistent. Practical experience and testing have shown that many small dequeue operations cause a lot of locking contention and as such badly affect performance. So it actually is a welcome enhancement to aquire the queue lock only once for the whole batch dequeue operation. As dequeing is a comperatively fast operation, the lock is not held for extended periods of time. + +A first approach to this functionality is shown in algorithm \ref{alg_ldeq_batch_1}. Note that $C_{mBatch}$ is the configured maximum number of elements inside a batch, $i$ is an index to address the objects inside the batch. + +\begin{figure}[h] +\begin{center} +\includegraphics[scale=0.6]{rsyslog_queue_pointers.jpeg} +\end{center} +\caption{\textbf{Queue Store Pointers}: boxes represent queue entries, colored boxes entries with objects. Objects in green are unprocessed, in blue are dequeued but not deleted and those in gray have already been deleted. White indicates not yet used entries. Gray objects may be overwritten at any time. Their entries are actually free, we have used the gray color primarily to indicate there once existed objects. Each queue pointer points to the next entry to process.} +\label{fig_queue_ptr} +\end{figure} + +\begin{algorithm} +\caption{dequeueBatch($b$)} +\begin{algorithmic} +\label{alg_ldeq_batch_1} +\STATE lock queue mutex +\STATE $0 \to i$ +\WHILE{queue non-empty and $i < C_{mBatch}$} + \STATE obtain next obj $o$ from queue store + \STATE advance logical dequeue position + \STATE put $o$ into batch +\ENDWHILE +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +A key concept is somewhat hidden in \marginpar{queue pointers} \emph{advance logical dequeue position}. Each queue store is purely sequential, with objects being enqueued at one ``end'' of the store and dequeued at the other. Of course, each queue store has only finite capacity, but we ignore this to explain the overall picture. A queue can be implemented by two pointers: one that points to the tail of the queue, where new messages are enqueued and one that points to the head of it, where new messages are dequeued. The idea is now to duplicate the dequeue pointer and split it into one for (logical) dequeue and one for deletion. Figure \ref{fig_queue_ptr} shows this three-pointer approach. Now, we can simple advance either the dequeue or deletion pointer, depending on operation, and do not need to find the first dequeue position inside the queue store. The dequeue pointer always points at it. This mode can be implemented with all currently existing queue storage drivers (but the sequential disk driver may need to use a second file handle or stream object instead of two pointers). + +This makes an efficient implementation of algorithm \ref{alg_ldeq_batch_1} possible: when it logically dequeues, it just needs to advance the dequeue pointer. So the algorithm executes in $O(n)$ time where $n$ specifies the number of elements to dequeue with an upper bound of $C_{mBatch}$. + +\begin{figure}[h] +\begin{center} +\includegraphics[scale=0.6]{rsyslog_queue_pointers2.jpeg} +\end{center} +\caption{\textbf{Physically Dequeueing Messages}: In this sample, we have two batches. With multiple workers, they may be deleted in any order.} +\label{fig_queue_ptr_deq} +\end{figure} + +Furthermore, we can also improve algorithm \ref{alg_pdeq_batch_1}: Consider that each batch is logically dequeued as an atomic operation. That means all batch objects form a sequential subset of the queue. Figure \ref{fig_queue_ptr_deq} shows the situation when two batches have been dequeued. So the costly ``find'' operation now needs to be carried out only once at the beginning of the batch. As all other objects are sequential, once we have found the batch begin inside the queue, we can simply delete the $|b|$ elements in queue-sequential order after it. So the cost of the find operation can be reduced from $O(|b|)$ to $O(1)$. + +We can even reduce the remaining cost of the find operation. If the batch to be deleted is right at the queue's head (as is ``B1'' in the figure), the ``find'' immediately terminates with the first element and incurs no cost at all. The situation is different if the batch is not at the queue head, ``B2'' is an example for that (assuming that ``B1'' has not yet been dequeued). We would now still need to search over the objects that are not part of the batch and can then finally get to the object at the head of the batch in question. For queue storage drivers that support random access to queue elements, storing a simple pointer to the batches' queue head element further improves the situation and enables $O(1)$ access to the queue element. This is indicated by the dotted lines in figure \ref{fig_queue_ptr_deq}. Once the head of the queue has been found, two things can happen (depending on the capabilities of the queue storage driver): + +\begin{enumerate} +\item the head element can be flagged as ``this and next $n$ elements are deleted'' +\item all elements are actually deleted +\end{enumerate} + +Note that a mixed form is also possible (and probably useful for our \emph{singly} linked list storage driver: there, some $n'$ elements be actually deleted and the head element is flagged as ``this and next $n - n'$ elements are deleted''. Note that in the linked-list case, all but the first elements can be deleted with ease\footnote{It can be considered to change from a singly-linked list to a doubly-linked list, if the benefit outweighs the extra effort required.}, so probably just the head would stay inside the queue. Note that removing elements off the queue, where possible, is useful because it frees resources. On a busy system, freeing messages as soon as possible can prevent message loss (in non-audit-grade setup) or system slowdown. So it should be done when possible. + +If we have a purely sequential queue storage driver (currently the sequential disk driver), finding and updating the head element is not an option. Even in this case, we can observe that the batch at the actual deletion pointer will eventually be submitted for deletion. So a route to take is to create a list of elements that can be deleted as soon as the physical dequeue pointer reaches any of these elements. We call this the \marginpar{to-delete list}``to-delete list''. To facilitate processing, this list must be ordered in sequence of dequeing. This information may not be available from the storage subsystem itself, but it can easily be generated. To do so, a strictly monotonically increasing counter is kept with each logical dequeue operation and stored as part of the batch\footnote{As this must be done via the usual computer-implemented modular arithmetic, we must be careful that we do not see repetion of values because of overflows. Each day has $60 \cdot 60 \cot 24 = 86,400$ seconds (ignoring the subleties of UTC). Now let's assume that we have a moderately-busy system with 1,000 messages per second. We further assume, to be on the save side, that each message is processed inside its own batch. So we have $86,400,000$ batches per day. If we now use a typical $32$-bit integer for generating the batch IDs, we the unique range will be used up after +$$\frac{2^{32}}{8640000} \approx 497 \text{ days}$$ +days of uninterrupted rsyslog operation. While this sounds somewhat save, it goes down to approximately 10 days of messages are submitted at rate of 50,000 messages per second (which is high, but not unheared of). So it is strongly advised to use 64 bits, which we consider to be save, because for our 1,000 messages per second the range would be exhausted only after +$$\frac{2^{64}}{8640000} \approx 2.135 \cdot 10^{11} \text{ days}$$ +which equals approximately $584,500,000$ \emph{years}. So even at a rate of one million messages per second, the range would be sufficient for over 500,000 years of continuos operations -- that should be far sufficient.} +An example: let us assume that ``B2'' was submitted for deletion first. Then, the head of ``B2'' is not at the queue's delete pointer. As such, no action can be carried out immediately. So the batch head pointer is stored into a ``to be deleted'' list. Processing continues. Some time later, batch ``B1'' is submitted for deletion. Now, the head pointer is at the head of the delete list, as such all batch elements are dequeued. Then, the ``to be deleted'' list is checked, and ``B2'' is found in it. Now, ``B2'' is at the head of the (new) deletion pointer and can also be removed. So, ultimately, all messages are physically dequeued. This is more formally describe in algorithm \ref{alg_phys_deq_seq_store}. In that pseudocode, we made a simplification by always putting the to be deleted batch in the ``to-delete'' list, which then enables us to use somewhat more generic code to carry out the work. + +Note that there is a price to pay for deletions via the ``to-delete'' list: if a sudden fatal failure happens during processing, the set of duplicate messages is increased. For example, if a fatal failure happens after ``B2'' has been fully processed and scheduled for deletion, but \emph{before ``B1'' is also submitted for deletion}, ``B2'' will be reprocessed after recovery. This would not happen if ``B2'' would have been removed from the queue. + +\begin{algorithm} +\caption{deleteBatch($b$)} +\begin{algorithmic} +\label{alg_phys_deq_seq_store} +\REQUIRE queue mutex is locked by caller +\STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ +\COMMENT ``to-delete'' list must be in order of logical dequeue +\WHILE{$D.head = Q.deletePtr$} + \FOR{$|b|$ elements} + \STATE delete element at queue head + \STATE move $q.deletePtr$ + \ENDFOR + \STATE remove head of ``to-delete'' list +\ENDWHILE +\end{algorithmic} +\end{algorithm} + +\paragraph{Warp-Up of Queue Delete Operations} +When evaluating which route to take, the ``to-delete'' list approach looks elegant for all cases. The negative side effect of potentially increased message duplication currently does not even exist: today, the sequential disk queue storage driver permits only a single worker thread and thus there always will be only one thread at a time. Even if we remove that limitation, message duplication could not be avoided, as stated in the algorithm description above. What remains are the other queue storage drivers. However, they operate in-memory, so message duplication will not happen simply because all messages will be lost on sudden fatal failure. The advantage of limited message duplication only exists in the so-far hypothetical case of a random-access, audit-grade disk queue storage driver. Thus, the decision could be postponed unless that happens (if it ever does). + +From a code complexity point of view, the ``to-delete'' list approch is definitely advantagous. Not only because of the reduced number of algorithms required. We also do not need to maintain unique batch IDs and all the logic associated with them. + +The other aspect to look at is memory consumption. Assuming that we delete the actual objects, just not their containers inside the queue, extra memory consumption is not really that worse. More importantly, currently only the linked-list queue storage driver can benefit at all, because it is the only driver capable of deleting queue entries in mid-queue. All others, including the array memory driver, do not have this capability. + +From a performance point of view, the ``to delete'' list approach looks approximately as good as the others, with some mild better performance for some storage drivers for a non-``to delete'' list approach. This can be mitigated, especially if the potentially somewhat-costly maintenance of the ``to-delete'' list is slightly optimized and the algorithm actually checks if the to be deleted batch is right at the queue's delete pointer position. The improved code simplicity, together with current CPU's code caching, may even result in an otherwise not expected speedup. + +In conclusion, we will implement the ``to-delete'' list approach on the queue layer (above the queue storage drivers). However, we will leave the window open to permit overwriting it with queue storage driver specific functionality. How to do this will not be specified now, as there is currently no need and we do not even know if there ever will be. However, we retain the discussion on the various modes as well as the relevant algorithmic discussions and data structurs inside this paper so that it is readily available should need arise. We also think this is important so that everybody later knows that the decision was made based on good argument and not by accident (we consider this useful in another design enhancement attempt). + +\paragraph{Processing Sequence} Looking at the processing sequence, we notice that always objects are dequeued, then processed and then deleted. Then, the whole process starts again. In particular, this meanss that after the previous batch has been deleted, the next batch will be dequeued. Now consider that we need to have exclusive access to the queue for both of these operations. As such it seems natural to combine this into a single step, further reducing potential locking contention. + +Note that a side-effect of this approach is that messages can be deleted only when a new batch is dequeued. With current design, this means that at least one message must reside inside the queue. Otherwise, the last batch will not be deleted. However, this something that can (and must!) be solved on the queue worker layer, in that it deletes a batch when the queue is empty. + +This leads us to the implementation of dequeueBatch() and deleteBatch() shown in algorithms \ref{alg_deq_batch_final} and \ref{alg_del_batch_final}. Note that $l$ is a flag variable that indicates if the queue is already locked. + +\begin{algorithm} +\caption{dequeueBatch($b$): final version} +\begin{algorithmic} +\label{alg_deq_batch_final} +\STATE lock queue mutex +\STATE call deleteBatch(b, 1) +\STATE $0 \to i$ +\WHILE{queue non-empty and $i < C_{mBatch}$} + \STATE obtain next obj $o$ from queue store + \STATE advance dequeue position + \STATE put $o$ into batch +\ENDWHILE +\STATE commit queue changes to storage system (if needed, e.g. fsync()) +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + + +\begin{algorithm} +\caption{deleteBatch($b, l$): final version} +\begin{algorithmic} +\label{alg_del_batch_final} +\IF{queue not yet locked (test via $l$)} + \STATE lock queue mutex +\ENDIF +\FORALL{objects $o$ in $b$} + \STATE destruct $o$ +\ENDFOR +\STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ +\COMMENT ``to-delete'' list must be in order of logical dequeue +\WHILE{$D.head = Q.deletePtr$} + \FOR{$|b|$ elements} + \STATE delete element at queue head + \STATE move $q.deletePtr$ + \ENDFOR + \STATE remove head of ``to-delete'' list +\ENDWHILE +\STATE commit queue changes to storage system (if needed, e.g. fsync()) +\IF{queue not yet locked (test via $l$)} + \STATE unlock queue mutex +\ENDIF +\end{algorithmic} +\end{algorithm} + +\subsubsection{Queue Stores} +Currently, rsyslog supports three different types of queue store drivers: + +\begin{itemize} +\item memory array +\item memory linked list +\item disk sequential file +\end{itemize} + +They all provide an abstracted sequential queue store as shown in figure \ref{fig_queue_ptr} on page \pageref{fig_queue_ptr}. + +Obviously, some differences exist. Most importantly, the disk sequential file driver does \emph{not} support more than one queue worker thread (in order to prevent excessive disk activity and the subtle issues with rewriting parts of sequential files). So if this driver is used, the queue automatically limits itself to a maximum of one worker thread (even if user configuration settings + +Different queue store drivers have different properties: + +\begin{tabular}{|l||l|l|l|}\hline + & array & linked list & seqential file \\ \hline +pointer type & integer index & memory address & file number and \\ + & & & offset within file \\ \hline +physical access & random & random & sequential \\ \hline +remove middle & no & yes & no \\ +elements & & & \\ \hline +access to $n$-th& $O(1)$, index:& $O(n)$, follow & not supported \\ +element & $n \mod C_{mMsg}$ & pointer links & \\ \hline +speed & fastest & fast & slow \\\hline +mem overhead & large & some & almost none \\\hline +reliability & reliable & reliable & audit-grade\footnote{if configured correctly}\\ +\hline +\end{tabular} + +\subsubsection{Implementation} +The actual implementation will be based on algorithms \ref{alg_deq_batch_final} and \ref{alg_del_batch_final}. The rsyslog v3 queue storage driver will be extended one additional method, which permits non-destructive dequeueing of elements. As such, the driver now has the $qAdd()$, $qDeq()$, and $qDel()$ entry points (together with the usual construction and destruction entry points). The queue drivers must support the three pointers for enqueue, dequeue and delete. The ``to-delete'' list will be maintained on the upper queue layer (and not the queue driver layer). This functionality will be optimized so that if a batch to delete is right at the queue's delete pointer, it will immediatly be deleted and not be sent to the ``to-delete'' list. This is especially important with the sequential disk driver, as the condition here always is true (and thus the driver can pretend this in the relevant API without even comparing any pointers -- what would otherwise quite complicated in this driver. + +The full list of the queue store driver interface is: + +\paragraph{qConstruct} Initializes the queue store. + +\paragraph{qDestruct} Destructs the queue store, including all messages that may still be present in it. + +\paragraph{qAdd} Enqueue a new object into the queue. Note that this entry point must only be called when the queue is non-full. + +\paragraph{qDeq} Non-destructive dequeue of the object at queue head. Dequeue pointer is advanced. + +\paragraph{qDel} Delete the object at queue head. Delete pointer is advanced. + +Disk queue store drivers may support additional internal functions. However, they should not be exposed to the rest of the queue subsystem. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{queue_msg_state.jpeg} +\end{center} +\caption{Logical Message States during Queue Processing} +\label{fig_queue_msg_state} +\end{figure} + +Figure \ref{fig_queue_msg_state} shows a logical message state diagram during queue processing. There is no actual state variable, but rather the processing flow demands these state. Note that the state transition from ``dequeued'' to ``queued'' only happens after a fatal failure and a successful system recovery. So this is a rather exceptional case. + +Another subtle issue is that we now need two different queue size counters: one for seeing when the queue is physically full and one for detecting when there are no more messages to be dequeued. + +As a simplification, support for ungetting objects can be removed (as objects never leave the queue), what also means that cancel-processing is probably less complex. + +\paragraph{Sequential Disk Queue Store Driver} +The enequeue, deqeueue and delete pointers must be implemented via three stream objects. Most importantly, the dequeue stream must be configured not to delete files when it closes them. A side-effect of this implementation is that data is actually read twice, once to actually obtain it and a second time to delete it. This could only be avoided by an overall redesign on how the disk queue works. + +\subsubsection{Checkmarks} +The following things need to be verified in the actual implementation. + +\paragraph{Queue Full} +Is it possible to set an infinte timeout on queue full condition during enqueue? If not, we must provide it. + +\paragraph{Termination the Queue} +If we cancel a worker, we need to start from the physical dequeue pointer and pull everything that is not scheduled for deletion - NOT from the logical dequeue pointer. + +\paragraph{Failed Messages} +If a message fails on a detached action queue, no backup processing is available (because we detect the failure at a point where the message is already considered processed from the main queue's point of view. We need address this and have two options: + + +I see two approaches at handling this: + +a) we enable an action to configure a backup file that shall receive all +message permanent failures. This is simple (not only to implement but to +configure and understand) + +b) we push the failed message back to the main queue, but with an indication +that it failed in an action. This is harder to implement and most importantly +harder to understand/configure, but more flexible + +\section{Future Development} +This section covers topics that can not currently be developed, but where important thoughts came up in discussions. For obvious reasons, the section has brainstorming character. + +\subsection{Audit-Grade High Performance Queue Storage Driver} +An audit grade driver must ensure that no message is lost, but should also be able to handle large workloads. The sequential disk driver does not support the later. + +An additional disk driver is envisioned with the properties like the linked list driver, but a reliable on-disk store. In particular, random access to queue elements is desired, which requires an addressing capability. + +A potential implementation requires a pre-formatted file. That file is organized in pages of $n$ bytes (e.g. 1K). The page index is used to address a queue item. If an item fits into 1K, it uses one page. If it is larger than 1K, consequtive pages are used to store the element. A page header must be present to indicate how many pages a single element is made up of. + +It may be noted that we could even improve performance by keeping part of the data in-memory. For audit-gradeness, it is required that upon enqueue the message is written to disk and only after final processing it needs to be removed. However, it is not forbidden to keep the same message in main memory. That way, the logical dequeue operation could be done one the in-memory representation. Only the physical dequeue would need to write to disk again. As such, we save one disk read out of three writes and one read otherwise required (so one can roughly say that we save one third of disk operations. + +Note that due to potential multi-pages messages we can not directly address individual elements, but we can reliably and quikly address elements whom's address we know (learned, for example, during logical dequeue). This is similar to the organization of the in-memory linked list. Actally, such a store \emph{is} a linked list implementation, just that memory is allocated on disk instead of in main memory. + +To further improve speed, object representation could be zipped before being written to a page. + +File Layout +Page 0: control structures (most importantyle queue pointers) (can make sense to store in a separate file, which could be moved to a dedicated disk subsystem - can potentially greatly reduce disk seek times). +Page 1 to n: actual object storage + +Algorithms \ref{alg_AuditGradeStoreEnqueue} and \ref{alg_AuditGradeStoreDelete} show how records are enqueued and deleted. Note that the delete part does not even need to read back the record. If we keep at last some records in-memory, the performance cost of ultra-reliable mode can actually comparatively low. Note that we may not even really need to commit data to the storage system in ``AuditGradeStoreDelete()'', because if a fatal failure occurs at this point, at worst message duplication may happen, what we have considered to be acceptable. + +\begin{algorithm} +\caption{AuditGradeStoreEnqueue($o$)} +\begin{algorithmic} +\label{alg_AuditGradeStoreEnqueue} +\REQUIRE queue mutex is locked by caller +\STATE write $o$ to current enqueue location +\STATE update \& write queue structures [page 0] +\STATE sync all files touched +\STATE store $o$ in an in-memory structure (or a cache) +\end{algorithmic} +\end{algorithm} + +\begin{algorithm} +\caption{AuditGradeStoreDelete($o$)} +\begin{algorithmic} +\label{alg_AuditGradeStoreDelete} +\REQUIRE queue mutex is locked by caller +\STATE update queue dequeue pointer \& write queue structures [page 0] +\STATE sync all files touched +\end{algorithmic} +\end{algorithm} + + +\end{document} diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html new file mode 100644 index 00000000..63c186a3 --- /dev/null +++ b/doc/dev_oplugins.html @@ -0,0 +1,331 @@ +<html> +<head> +<title>writing rsyslog output plugins (developer's guide)</title> +</head> +<body> +<h1>Writing Rsyslog Output Plugins</h1> +<p>This page is the begin of some developer documentation for writing output +plugins. Doing so is quite easy (and that was a design goal), but there currently +is only sparse documentation on the process available. I was tempted NOT to +write this guide here because I know I will most probably not be able to +write a complete guide. +<p>However, I finally concluded that it may be better to have same information +and pointers than to have nothing. +<h2>Getting Started and Samples</h2> +<p>The best to get started with rsyslog plugin development is by looking at +existing plugins. All that start with "om" are <b>o</b>utput <b>m</b>odules. That +means they are primarily thought of being message sinks. In theory, however, output +plugins may aggergate other functionality, too. Nobody has taken this route so far +so if you would like to do that, it is highly suggested to post your plan on the +rsyslog mailing list, first (so that we can offer advise). +<p>The rsyslog distribution tarball contains two plugins that are extremely well +targeted for getting started: +<ul> +<li>omtemplate +<li>omstdout +</ul> +Plugin omtemplate was specifically created to provide a copy template for new output +plugins. It is bare of real functionality but has ample comments. Even if you decide +to start from another plugin (or even from scratch), be sure to read omtemplate source +and comments first. The omstdout is primarily a testing aide, but offers support for +the two different parameter-passing conventions plugins can use (plus the way to +differentiate between the two). It also is not bare of functionaly, only mostly +bare of it ;). But you can actually execute it and play with it. +<p>In any case, you should also read the comments in ./runtime/module-template.h. +Output plugins are build based on a large set of code-generating macros. These +macros handle most of the plumbing needed by the interface. As long as no +special callback to rsyslog is needed (it typically is not), an output plugin does +not really need to be aware that it is executed by rsyslog. As a plug-in programmer, +you can (in most cases) "code as usual". However, all macros and entry points need to be +provided and thus reading the code comments in the files mentioned is highly suggested. +<p>In short, the best idea is to start with a template. Let's assume you start by +copying omtemplate. Then, the basic steps you need to do are: +<ul> +<li>cp ./plugins/omtemplate ./plugins/your-plugin +<li>mv cd ./plugins/your-plugin +<li>vi Makefile.am, adjust to your-plugin +<li>mv omtemplate.c your-plugin.c +<li>cd ../.. +<li>vi Makefile.am configure.ac +<br>search for omtemplate, copy and modify (follow comments) +</ul> +<p>Basically, this is all you need to do ... Well, except, of course, coding +your plugin ;). For testing, you need rsyslog's debugging support. Some useful +information is given in "<a href="troubleshoot.html">troubleshooting rsyslog</a> +from the doc set. +<h2>Special Topics</h2> +<h3>Threading</h3> +<p>Rsyslog uses massive parallel processing and multithreading. However, a plugin's entry +points are guaranteed to be never called concurrently <b>for the same action</b>. +That means your plugin must be able to be called concurrently by two or more +threads, but you can be sure that for the same instance no concurrent calls +happen. This is guaranteed by the interface specification and the rsyslog core +guards against multiple concurrent calls. An instance, in simple words, is one +that shares a single instanceData structure. +<p>So as long as you do not mess around with global data, you do not need +to think about multithreading (and can apply a purely sequential programming +methodology). +<p>Please note that duringt the configuraton parsing stage of execution, access to +global variables for the configuration system is safe. In that stage, the core will +only call sequentially into the plugin. +<h3>Getting Message Data</h3> +<p>The doAction() entry point of your plugin is provided with messages to be processed. +It will only be activated after filtering and all other conditions, so you do not need +to apply any other conditional but can simply process the message. +<p>Note that you do NOT receive the full internal representation of the message +object. There are various (including historical) reasons for this and, among +others, this is a design decision based on security. +<p>Your plugin will only receive what the end user has configured in a $template +statement. However, starting with 4.1.6, there are two ways of receiving the +template content. The default mode, and in most cases sufficient and optimal, +is to receive a single string with the expanded template. As I said, this is usually +optimal, think about writing things to files, emailing content or forwarding it. +<p>The important philosophy is that a plugin should <b>never</b> reformat any +of such strings - that would either remove the user's ability to fully control +message formats or it would lead to duplicating code that is already present in the +core. If you need some formatting that is not yet present in the core, suggest it +to the rsyslog project, best done by sending a patch ;), and we will try hard to +get it into the core (so far, we could accept all such suggestions - no promise, though). +<p>If a single string seems not suitable for your application, the plugin can also +request access to the template components. The typical use case seems to be databases, where +you would like to access properties via specific fields. With that mode, you receive a +char ** array, where each array element points to one field from the template (from +left to right). Fields start at arrray index 0 and a NULL pointer means you have +reached the end of the array (the typical Unix "poor man's linked list in an array" +design). Note, however, that each of the individual components is a string. It is +not a date stamp, number or whatever, but a string. This is because rsyslog processes +strings (from a high-level design look at it) and so this is the natural data type. +Feel free to convert to whatever you need, but keep in mind that malformed packets +may have lead to field contents you'd never expected... +<p>If you like to use the array-based parameter passing method, think that it +is only available in rsyslog 4.1.6 and above. If you can accept that your plugin +will not be working with previous versions, you do not need to handle pre 4.1.6 cases. +However, it would be "nice" if you shut down yourself in these cases - otherwise the +older rsyslog core engine will pass you a string where you expect the array of pointers, +what most probably results in a segfault. To check whether or not the core supports the +functionality, you can use this code sequence: +<pre> +<code> +BEGINmodInit() + rsRetVal localRet; + rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts); + unsigned long opts; + int bArrayPassingSupported; /* does core support template passing as an array? */ +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + /* check if the rsyslog core supports parameter passing code */ + bArrayPassingSupported = 0; + localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts); + if(localRet == RS_RET_OK) { + /* found entry point, so let's see if core supports array passing */ + CHKiRet((*pomsrGetSupportedTplOpts)(&opts)); + if(opts & OMSR_TPL_AS_ARRAY) + bArrayPassingSupported = 1; + } else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) { + ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */ + } + DBGPRINTF("omstdout: array-passing is %ssupported by rsyslog core.\n", bArrayPassingSupported ? "" : "not "); + + if(!bArrayPassingSupported) { + DBGPRINTF("rsyslog core too old, shutting down this plug-in\n"); + ABORT_FINALIZE(RS_RET_ERR); + } + +</code> +</pre> +<p>The code first checks if the core supports the OMSRgetSupportedTplOpts() API (which is +also not present in all versions!) and, if so, queries the core if the OMSR_TPL_AS_ARRAY mode +is supported. If either does not exits, the core is too old for this functionality. The sample +snippet above then shuts down, but a plugin may instead just do things different. In +omstdout, you can see how a plugin may deal with the situation. +<p><b>In any case, it is recommended that at least a graceful shutdown is made and the +array-passing capability not blindly be used.</b> In such cases, we can not guard the +plugin from segfaulting and if the plugin (as currently always) is run within +rsyslog's process space, that results in a segfault for rsyslog. So do not do this. +<h3>Batching of Messages</h3> +<p>Starting with rsyslog 4.3.x, batching of output messages is supported. Previously, only +a single-message interface was supported. +<p>With the <b>single message</b> plugin interface, each message is passed via a separate call to the plugin. +Most importantly, the rsyslog engine assumes that each call to the plugin is a complete transaction +and as such assumes that messages be properly commited after the plugin returns to the engine. +<p>With the <b>batching</b> interface, rsyslog employs something along the line of +"transactions". Obviously, the rsyslog core can not make non-transactional outputs +to be fully transactional. But what it can is support that the output tells the core which +messages have been commited by the output and which not yet. The core can than take care +of those uncommited messages when problems occur. For example, if a plugin has received +50 messages but not yet told the core that it commited them, and then returns an error state, the +core assumes that all these 50 messages were <b>not</b> written to the output. The core then +requeues all 50 messages and does the usual retry processing. Once the output plugin tells the +core that it is ready again to accept messages, the rsyslog core will provide it with these 50 +not yet commited messages again (actually, at this point, the rsyslog core no longer knows that +it is re-submiting the messages). If, in contrary, the plugin had told rsyslog that 40 of these 50 +messages were commited (before it failed), then only 10 would have been requeued and resubmitted. +<p>In order to provide an efficient implementation, there are some (mild) constraints in that +transactional model: first of all, rsyslog itself specifies the ultimate transaction boundaries. +That is, it tells the plugin when a transaction begins and when it must finish. The plugin +is free to commit messages in between, but it <b>must</b> commit all work done when the core +tells it that the transaction ends. All messages passed in between a begin and end transaction +notification are called a batch of messages. They are passed in one by one, just as without +transaction support. Note that batch sizes are variable within the range of 1 to a user configured +maximum limit. Most importantly, that means that plugins may receive batches of single messages, +so they are required to commit each message individually. If the plugin tries to be "smarter" +than the rsyslog engine and does not commit messages in those cases (for example), the plugin +puts message stream integrity at risk: once rsyslog has notified the plugin of transacton end, +it discards all messages as it considers them committed and save. If now something goes wrong, +the rsyslog core does not try to recover lost messages (and keep in mind that "goes wrong" +includes such uncontrollable things like connection loss to a database server). So it is +highly recommended to fully abide to the plugin interface details, even though you may +think you can do it better. The second reason for that is that the core engine will +have configuration settings that enable the user to tune commit rate to their use-case +specific needs. And, as a relief: why would rsyslog ever decide to use batches of one? +There is a trivial case and that is when we have very low activity so that no queue of +messages builds up, in which case it makes sense to commit work as it arrives. +(As a side-note, there are some valid cases where a timeout-based commit feature makes sense. +This is also under evaluation and, once decided, the core will offer an interface plus a way +to preserve message stream integrity for properly-crafted plugins). +<p>The second restriction is that if a plugin makes commits in between (what is perfectly +legal) those commits must be in-order. So if a commit is made for message ten out of 50, +this means that messages one to nine are also commited. It would be possible to remove +this restriction, but we have decided to deliberately introduce it to simpify things. +<h3>Output Plugin Transaction Interface</h3> +<p>In order to keep compatible with existing output plugins (and because it introduces +no complexity), the transactional plugin interface is build on the traditional +non-transactional one. Well... actually the traditional interface was transactional +since its introduction, in the sense that each message was processed in its own +transaction. +<p>So the current <code>doAction()</b> entry point can be considered to have this +structure (from the transactional interface point of view): +<p><pre><code> +doAction() + { + beginTransaction() + ProcessMessage() + endTransaction() + } + </code></pre> +<p>For the <b>transactional interface</b>, we now move these implicit <code>beginTransaction()</code> +and <code>endTransaction(()</code> call out of the message processing body, resulting is such +a structure: +<p><pre><code> +beginTransaction() + { + /* prepare for transaction */ + } + +doAction() + { + ProcessMessage() + /* maybe do partial commits */ + } + +endTransaction() + { + /* commit (rest of) batch */ + } +</code></pre> +<p>And this calling structure actually is the transactional interface! It is as simple as this. +For the new interface, the core calls a <code>beginTransaction()</code> entry point inside the +plugin at the start of the batch. Similarly, the core call <code>endTransaction()</code> at the +end of the batch. The plugin must implement these entry points according to its needs. +<p>But how does the core know when to use the old or the new calling interface? This is rather +easy: when loading a plugin, the core queries the plugin for the <code>beginTransaction()</code> +and <code>endTransaction()</code> entry points. If the plugin supports these, the new interface is +used. If the plugin does not support them, the old interface is used and rsyslog implies that +a commit is done after each message. Note that there is no special "downlevel" handling +necessary to support this. In the case of the non-transactional interface, rsyslog considers +each completed call to <code>doAction</code> as partial commit up to the current message. +So implementation inside the core is very straightforward. +<p>Actually, <b>we recommend that the transactional entry points only be defined by those +plugins that actually need them</b>. All others should not define them in which case +the default commit behaviour inside rsyslog will apply (thus removing complexity from the +plugin). +<p>In order to support partial commits, special return codes must be defined for +<code>doAction</code>. All those return codes mean that processing completed successfully. +But they convey additional information about the commit status as follows: +<p> +<table border="0"> +<tr> +<td valign="top"><i>RS_RET_OK</i></td> +<td>The record and all previous inside the batch has been commited. +<i>Note:</i> this definition is what makes integrating plugins without the +transaction being/end calls so easy - this is the traditional "success" return +state and if every call returns it, there is no need for actually calling +<code>endTransaction()</code>, because there is no transaction open).</td> +</tr> +<tr> +<td valign="top"><i>RS_RET_DEFER_COMMIT</i></td> +<td>The record has been processed, but is not yet commited. This is the +expected state for transactional-aware plugins.</td> +</tr> +<tr> +<td valign="top"><i>RS_RET_PREVIOUS_COMMITTED</i></td> +<td>The <b>previous</b> record inside the batch has been committed, but the +current one not yet. This state is introduced to support sources that fill up +buffers and commit once a buffer is completely filled. That may occur halfway +in the next record, so it may be important to be able to tell the +engine the everything up to the previouos record is commited</td> +</tr> +</table> +<p>Note that the typical <b>calling cycle</b> is <code>beginTransaction()</code>, +followed by <i>n</i> times +<code>doAction()</code></n> followed by <code>endTransaction()</code>. However, if either +<code>beginTransaction()</code> or <code>doAction()</code> return back an error state +(including RS_RET_SUSPENDED), then the transaction is considered aborted. In result, the +remaining calls in this cycle (e.g. <code>endTransaction()</code>) are never made and a +new cycle (starting with <code>beginTransaction()</code> is begun when processing resumes. +So an output plugin must expect and handle those partial cycles gracefully. +<p><b>The question remains how can a plugin know if the core supports batching?</b> +First of all, even if the engine would not know it, the plugin would return with RS_RET_DEFER_COMMIT, +what then would be treated as an error by the engine. This would effectively disable the +output, but cause no further harm (but may be harm enough in itself). +<p>The real solution is to enable the plugin to query the rsyslog core if this feature is +supported or not. At the time of the introduction of batching, no such query-interface +exists. So we introduce it with that release. What the means is if a rsyslog core can +not provide this query interface, it is a core that was build before batching support +was available. So the absence of a query interface indicates that the transactional +interface is not available. One might now be tempted the think there is no need to do +the actual check, but is is recommended to ask the rsyslog engine explicitely if +the transactional interface is present and will be honored. This enables us to +create versions in the future which have, for whatever reason we do not yet know, no +support for this interface. +<p>The logic to do these checks is contained in the <code>INITChkCoreFeature</code> macro, +which can be used as follows: +<p><pre><code> +INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); +</code></pre> +<p>Here, bCoreSupportsBatching is a plugin-defined integer which after execution is +1 if batches (and thus the transational interface) is supported and 0 otherwise. +CORE_FEATURE_BATCHING is the feature we are interested in. Future versions of rsyslog +may contain additional feature-test-macros (you can see all of them in +./runtime/rsyslog.h). +<p>Note that the ompsql output plugin supports transactional mode in a hybrid way and +thus can be considered good example code. + +<h2>Open Issues</h2> +<ul> +<li>Processing errors handling +<li>reliable re-queue during error handling and queue termination +</ul> + + + +<h3>Licensing</h3> +<p>From the rsyslog point of view, plugins constitute separate projects. As such, +we think plugins are not required to be compatible with GPLv3. However, this is +no legal advise. If you intend to release something under a non-GPLV3 compatible license +it is probably best to consult with your lawyer. +<p>Most importantly, and this is definite, the rsyslog team does not expect +or require you to contribute your plugin to the rsyslog project (but of course +we are happy if you do). +<h2>Copyright</h2> +<p>Copyright (c) 2009 <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> +and <a href="http://www.adiscon.com/en/">Adiscon</a>.</p> +<p>Permission is granted to copy, distribute and/or modify this document under +the terms of the GNU Free Documentation License, Version 1.2 or any later +version published by the Free Software Foundation; with no Invariant Sections, +no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be +viewed at <a href="http://www.gnu.org/copyleft/fdl.html"> +http://www.gnu.org/copyleft/fdl.html</a>.</p> +</body> +</html> diff --git a/doc/direct_queue0.png b/doc/direct_queue0.png Binary files differnew file mode 100644 index 00000000..6d1b373f --- /dev/null +++ b/doc/direct_queue0.png diff --git a/doc/direct_queue1.png b/doc/direct_queue1.png Binary files differnew file mode 100644 index 00000000..503f8151 --- /dev/null +++ b/doc/direct_queue1.png diff --git a/doc/direct_queue2.png b/doc/direct_queue2.png Binary files differnew file mode 100644 index 00000000..cbb99af8 --- /dev/null +++ b/doc/direct_queue2.png diff --git a/doc/direct_queue3.png b/doc/direct_queue3.png Binary files differnew file mode 100644 index 00000000..cc49299f --- /dev/null +++ b/doc/direct_queue3.png diff --git a/doc/direct_queue_directq.png b/doc/direct_queue_directq.png Binary files differnew file mode 100644 index 00000000..c5d8769d --- /dev/null +++ b/doc/direct_queue_directq.png diff --git a/doc/direct_queue_rsyslog.png b/doc/direct_queue_rsyslog.png Binary files differnew file mode 100644 index 00000000..6150222d --- /dev/null +++ b/doc/direct_queue_rsyslog.png diff --git a/doc/direct_queue_rsyslog2.png b/doc/direct_queue_rsyslog2.png Binary files differnew file mode 100644 index 00000000..807b064d --- /dev/null +++ b/doc/direct_queue_rsyslog2.png diff --git a/doc/droppriv.html b/doc/droppriv.html new file mode 100644 index 00000000..7293e872 --- /dev/null +++ b/doc/droppriv.html @@ -0,0 +1,60 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>dropping privileges in rsyslog</title> +</head> +<body> +<h1>Dropping privileges in rsyslog</h1> +<p><b>Available since: </b> 4.1.1</p> +<p><b>Description</b>:</p> +<p> +Rsyslogd provides the ability to drop privileges by +impersonating as another user and/or group after startup. + +<p>Please note that due to POSIX standards, rsyslogd always needs to start +up as root if there is a listener who must bind to a network port below 1024. +For example, the UDP listener usually needs to listen to 514 and as such +rsyslogd needs to start up as root. + +<p>If you do not need this functionality, you can start rsyslog directly as an ordinary +user. That is probably the safest way of operations. However, if a startup as +root is required, you can use the $PrivDropToGroup and $PrivDropToUser config +directives to specify a group and/or user that rsyslogd should drop to after initialization. +Once this happend, the daemon runs without high privileges (depending, of +course, on the permissions of the user account you specified). +<p>There is some additional information available in the +<a href="http://wiki.rsyslog.com/index.php/Security#Dropping_Privileges">rsyslog wiki</a>. +<p><b>Configuration Directives</b>:</p> +<ul> +<li><b>$PrivDropToUser</b><br> +Name of the user rsyslog should run under after startup. Please note that +this user is looked up in the system tables. If the lookup fails, privileges are +NOT dropped. Thus it is advisable to use the less convenient $PrivDropToUserID directive. +If the user id can be looked up, but can not be set, rsyslog aborts. +<br> +</li> +<li><b>$PrivDropToUserID</b><br> +Much the same as $PrivDropToUser, except that a numerical user id instead of a name +is specified.Thus, privilege drop will always happen. +rsyslogd aborts. +<li><b>$PrivDropToGroup</b><br> +Name of the group rsyslog should run under after startup. Please note that +this user is looked up in the system tables. If the lookup fails, privileges are +NOT dropped. Thus it is advisable to use the less convenient $PrivDropToGroupID directive. +Note that all supplementary groups are removed from the process if $PrivDropToGroup is +specified. +If the group id can be looked up, but can not be set, rsyslog aborts. +<br> +</li> +<li><b>$PrivDropToGroupID</b><br> +Much the same as $PrivDropToGroup, except that a numerical group id instead of a name +is specified. Thus, privilege drop will always happen. +</ul> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer +Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> + +</body></html> diff --git a/doc/expression.html b/doc/expression.html index e7eb7842..9e37cb7a 100644 --- a/doc/expression.html +++ b/doc/expression.html @@ -2,6 +2,7 @@ <html><head> <meta http-equiv="Content-Language" content="en"><title>Expressions</title></head> <body> +<a href="rsyslog_conf_filter.html">back</a> <h1>Expressions</h1> <p>Rsyslog supports expressions at a growing number of places. So far, they are supported for filtering messages.</p><p>Expression support is provided by RainerScript. For now, please see the formal expression definition in <a href="rainerscript.html">RainerScript ABNF</a>. It is the "expr" node.</p><p>C-like comments (/* some comment */) are supported <span style="font-weight: bold;">inside</span> the expression, but not yet in the rest of the configuration file.</p><p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] @@ -13,4 +14,4 @@ Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> -</body></html>
\ No newline at end of file +</body></html> diff --git a/doc/features.html b/doc/features.html index d221eb77..626ff65d 100644 --- a/doc/features.html +++ b/doc/features.html @@ -1,8 +1,8 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head><title>rsyslog features</title> - </head> <body> +<a href="rsyslog_conf.html">back</a> <h1>RSyslog - Features</h1> <p><b>This page lists both current features as well as those being considered for future versions of rsyslog.</b> If you @@ -95,13 +95,16 @@ via custom plugins</li> <li> an easy-to-write to plugin interface</li> <li> ability to send SNMP trap messages</li> <li> ability to filter out messages based on sequence of arrival</li> +<li>support for comma-seperated-values (CSV) output generation +(via the "csv" property replace option). The +CSV format supported is that from RFC 4180.</li> <li>support for arbitrary complex boolean, string and arithmetic expressions in message filters</li> </ul> <h2>World's first</h2> Rsyslog has an interesting number of "world's firsts" - things that were implemented for the first time ever in rsyslog. Some of them are still features not available elsewhere.<br><ul> -<li>world's first implementation of IETF I-D syslog-protocol (February 2006, version 1.12.2 and above)</li><li>world's first implementation of dynamic syslog on-the-wire compression (December 2006, version 1.13.0 and above)</li><li>world's first open-source implementation of a disk-queueing syslogd (January 2008, version 3.11.0 and above)</li> +<li>world's first implementation of IETF I-D syslog-protocol (February 2006, version 1.12.2 and above), now RFC5424</li><li>world's first implementation of dynamic syslog on-the-wire compression (December 2006, version 1.13.0 and above)</li><li>world's first open-source implementation of a disk-queueing syslogd (January 2008, version 3.11.0 and above)</li> <li>world's first implementation of IETF I-D syslog-transport-tls (May 2008, version 3.19.0 and above)</li> </ul> @@ -116,6 +119,13 @@ submit feature requests there (or via our forums). If we like them but they look quite long-lived (aka "not soon to be implemented"), they will possibly be migrated to this list here and at some time moved back to the bugzilla tracker.</p> +<p><b>Note that we also maintain a +<a href="http://www.rsyslog.com/sponsor_feature">list of features that are looking for sponsors</a>. +If you are interested in any of these features, or any other feature, you may consider sponsoring +the implementation. This is also a great way to show your commitment to the open source +community. Plus, it can be financially attractive: just think about how much less it may +be to sponsor a feature instead of purchasing a commercial implementation. Also, the benefit +of being recognised as a sponsor may even drive new customers to your business!</b> <ul> <li>port it to more *nix variants (eg AIX and HP UX) - this needs volunteers with access to those machines and knowledge </li> @@ -134,4 +144,15 @@ future of RFC 3195 in rsyslog</a>.</li> <p>To see when each feature was added, see the <a href="http://www.rsyslog.com/Topic4.phtml">rsyslog change log</a> (online only).</p> + +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> + </body></html> + diff --git a/doc/how2help.html b/doc/how2help.html index 0caa5a3a..4f0bd57a 100644 --- a/doc/how2help.html +++ b/doc/how2help.html @@ -1,59 +1,57 @@ -<html>
-<head>
-<title>How you can Help</title>
-</head>
-<body>
-<h2>How you can Help</h2>
-<p><b>You like rsyslog and would like to lend us a helping hand?</b> This page
-tells you how easy it is to help a little bit. You can contribute to the project
-even with a single mouse click! If you could pick a single item from the
-wish list, that would be awfully helpful!</p>
-<p>This is our wish list:</p>
-<ul>
- <li>let others know how great rsyslog is<ul>
- <li>rate us at <a href="http://freshmeat.net/rate/52985/">freshmeat.net</a>
- and <a href="http://www.icewalkers.com/vote.php?ID=2420">icewalkers.com</a></li>
- <li>spread word about rsyslog in forums and newsgroups</li>
- <li>place a link to <a href="http://www.rsyslog.com">www.rsyslog.com</a>
- from your home page</li>
- </ul>
- </li>
- <li>let us know about rsyslog - we are eager for feedback<ul>
- <li>tell us what you like and what you not like - so that we can include
- that into development</li>
- <li>tell us what you use rsyslog for - especially if you have high
- traffic volume or an otherwise "uncommon" deployment. We are looking for
- case studies and experience how rsyslog performs in unusual scenarios.</li>
- <li>allow us to post your thoughts and experiences as a "user story" on
- the web site (so far, none are there ;))</li>
- </ul>
- </li>
- <li>if you know how to create packages (rpm, deb, ...)<ul>
- <li>we would very much appreciate your help with package creation. We know
- that it is important to have good binary packages for a product to
- spread widely. Yet, we do not have the knowledge to do it all ourselves.
- <a href="mailto:rgerhards@adiscon.com">Drop Rainer a note </a>if you
- could help us out.</li>
- </ul>
- </li>
- <li>if you have configured a device for sending syslog data, and that device
- is not in our
- <a href="http://www.monitorware.com/en/syslog-enabled-products/">syslog
- configuration database</a>, you might want to tell us how to configure it.</li>
- <li>if you are a corporate user<ul>
- <li>you might consider <a href="http://www.adiscon.com">Adiscon</a>'s
- commercial <a href="http://www.monitorware.com/">MonitorWare products</a>
- for Windows, e.g. to deliver Windows Event Log data to rsyslogd (sales
- of the commercial products funds the open source development - and they
- also work very well).</li>
- <li>you might be interested in
- <a href="http://www.adiscon.com/Common/en/Products/techsup.php">
- purchasing professional support or add-on development</a> for rsyslog</li>
- </ul>
- </li>
-</ul>
-<p><b>We appreciate your help very much.</b> A big thank you for anything you
-might do!</p>
-
-</body>
-</html> +<html> +<head> +<title>How you can Help</title> +</head> +<body> +<h2>How you can Help</h2> +<p><b>You like rsyslog and would like to lend us a helping hand?</b> This page +tells you how easy it is to help a little bit. You can contribute to the project +even with a single mouse click! If you could pick a single item from the +wish list, that would be awfully helpful!</p> +<p>This is our wish list:</p> +<ul> + <li>let others know how great rsyslog is<ul> + <li>spread word about rsyslog in forums and newsgroups</li> + <li>place a link to <a href="http://www.rsyslog.com">www.rsyslog.com</a> + from your home page</li> + </ul> + </li> + <li>let us know about rsyslog - we are eager for feedback<ul> + <li>tell us what you like and what you not like - so that we can include + that into development</li> + <li>tell us what you use rsyslog for - especially if you have high + traffic volume or an otherwise "uncommon" deployment. We are looking for + case studies and experience how rsyslog performs in unusual scenarios.</li> + <li>allow us to post your thoughts and experiences as a "user story" on + the web site (so far, none are there ;))</li> + </ul> + </li> + <li>if you know how to create packages (rpm, deb, ...)<ul> + <li>we would very much appreciate your help with package creation. We know + that it is important to have good binary packages for a product to + spread widely. Yet, we do not have the knowledge to do it all ourselves. + <a href="mailto:rgerhards@adiscon.com">Drop Rainer a note </a>if you + could help us out.</li> + </ul> + </li> + <li>if you have configured a device for sending syslog data, and that device + is not in our + <a href="http://www.monitorware.com/en/syslog-enabled-products/">syslog + configuration database</a>, you might want to tell us how to configure it.</li> + <li>if you are a corporate user<ul> + <li>you might consider <a href="http://www.adiscon.com">Adiscon</a>'s + commercial <a href="http://www.monitorware.com/">MonitorWare products</a> + for Windows, e.g. to deliver Windows Event Log data to rsyslogd (sales + of the commercial products funds the open source development - and they + also work very well).</li> + <li>you might be interested in + <a href="http://www.adiscon.com/Common/en/Products/techsup.php"> + purchasing professional support or add-on development</a> for rsyslog</li> + </ul> + </li> +</ul> +<p><b>We appreciate your help very much.</b> A big thank you for anything you +might do!</p> + +</body> +</html diff --git a/doc/im3195.html b/doc/im3195.html index d6f2f2ed..aad9f3d1 100644 --- a/doc/im3195.html +++ b/doc/im3195.html @@ -4,6 +4,8 @@ </head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>RFC3195 Input Module</h1> <p><b>Module Name: im3195</b></p> <p><b>Author: </b>Rainer Gerhards diff --git a/doc/imfile.html b/doc/imfile.html index 5bdbce5c..af0413dd 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -2,6 +2,8 @@ <html><head> <meta http-equiv="Content-Language" content="en"><title>Text File Input Monitor</title></head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>Text File Input Module</h1> <p><b>Module Name: imfile</b></p> <p><b>Author: </b>Rainer Gerhards diff --git a/doc/imgssapi.html b/doc/imgssapi.html index d644303e..ec183fe7 100644 --- a/doc/imgssapi.html +++ b/doc/imgssapi.html @@ -4,6 +4,8 @@ </head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>GSSAPI Syslog Input Module</h1> <p><b>Module Name: imgssapi</b></p> <p><b>Author: </b>varmojfekoj</p> diff --git a/doc/imklog.html b/doc/imklog.html index b5b21e84..5bfab5ce 100644 --- a/doc/imklog.html +++ b/doc/imklog.html @@ -4,6 +4,8 @@ </head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>Kernel Log Input Module</h1> <p><b>Module Name: imklog</b></p> <p><b>Author: </b>Rainer Gerhards @@ -32,9 +34,9 @@ handled. The default is "off", in which case these messages are ignored. Switch it to on to submit non-kernel messages to rsyslog processing.<span style="font-weight: bold;"></span></li> <li><span style="font-weight: bold;"></span>$DebugPrintKernelSymbols -(imklog) [on/<b>off</b>]<br> +[on/<b>off</b>]<br> Linux only, ignored on other platforms (but may be specified)</li> -<li>$klogSymbolLookup (imklog) [on/<b>off</b>] -- +<li>$klogSymbolLookup [on/<b>off</b>] -- disables imklog kernel symbol translation (former klogd -x option). NOTE that this option is counter-productive on recent kernels (>= 2.6) because the kernel already does the symbol translation and this option breaks the information.<br> @@ -42,10 +44,19 @@ kernel already does the symbol translation and this option breaks the informatio it except if you have a very good reason. If you have one, let us know because otherwise new versions will no longer support it.<br> Linux only, ignored on other platforms (but may be specified)</li> -<li>$klogUseSyscallInterface (imklog) [on/<b>off</b>] +<li><b>$klogConsoleLogLevel</b> [<i>number</i>] +(former klogd -c option) -- sets the console log level. If specified, only messages with +up to the specified level are printed to the console. The default is -1, which means that +the current settings are not modified. To get this behavior, do not specify +$klogConsoleLogLevel in the configuration file. Note that this is a global parameter. Each time +it is changed, the previous definition is re-set. The one activate will be that one that is +active when imklog actually starts processing. In short words: do not specify this +directive more than once! +<br><b>Linux only</b>, ignored on other platforms (but may be specified)</li> +<li><b>$klogUseSyscallInterface</b> [on/<b>off</b>] -- former klogd -s option<br> Linux only, ignored on other platforms (but may be specified)</li> -<li>$klogSymbolsTwice (imklog) [on/<b>off</b>] -- +<li>$klogSymbolsTwice [on/<b>off</b>] -- former klogd -2 option<br> Linux only, ignored on other platforms (but may be specified)<br style="font-weight: bold;"> </li> @@ -67,7 +78,7 @@ is needed to start pulling kernel messages.<br> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer +Copyright © 2008-2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> diff --git a/doc/imrelp.html b/doc/imrelp.html index bfdaad84..53826ac2 100644 --- a/doc/imrelp.html +++ b/doc/imrelp.html @@ -4,6 +4,8 @@ </head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>RELP Input Module</h1> <p><b>Module Name: imrelp</b></p> <p><b>Author: Rainer Gerhards</b></p> diff --git a/doc/imtcp.html b/doc/imtcp.html index ecc72748..9ea7efa1 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -2,6 +2,8 @@ <html><head> <meta http-equiv="Content-Language" content="en"><title>TCP Syslog Input Module</title></head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>TCP Syslog Input Module</h1> <p><b>Module Name: imtcp</b></p> <p><b>Author: </b>Rainer Gerhards @@ -12,17 +14,43 @@ Encryption can be provided by using <a href="rsyslog_stunnel.html">stunnel</a> (an alternative is the use the <a href="imgssapi.html">imgssapi</a> modul).</p> -<p>In the future, multiple receivers may be configured by +<p>Multiple receivers may be configured by specifying -$InputTCPServerRun multiple times. This is not currently supported. +$InputTCPServerRun multiple times. This is available since version 4.3.1, earlier +versions do NOT support it. </p> <p><b>Configuration Directives</b>:</p> <ul> +<li>$InputTCPServerAddtlFrameDelimiter <Delimiter><br> +This directive permits to specify an additional frame delimiter for plain tcp syslog. +The industry-standard specifies using the LF character as frame delimiter. Some vendors, +notable Juniper in their NetScreen products, use an invalid frame delimiter, in Juniper's +case the NUL character. This directive permits to specify the ASCII value of the delimiter +in question. Please note that this does not guarantee that all wrong implementations can +be cured with this directive. It is not even a sure fix with all versions of NetScreen, +as I suggest the NUL character is the effect of a (common) coding error and thus will +probably go away at some time in the future. But for the time being, the value 0 can +probably be used to make rsyslog handle NetScreen's invalid syslog/tcp framing. +For additional information, see this +<a href="http://kb.monitorware.com/problem-with-netscreen-log-t1652.html">forum thread</a>. +<br><b>If this doesn't work for you, please do not blame the rsyslog team. Instead file +a bug report with Juniper!</b> +<br>Note that a similar, but worse, issue exists with Cisco's IOS implementation. They do +not use any framing at all. This is confirmed from Cisco's side, but there seems to be +very limited interest in fixing this issue. This directive <b>can not</b> fix the Cisco bug. +That would require much more code changes, which I was unable to do so far. Full details +can be found at the <a href="http://www.rsyslog.com/Article321.phtml">Cisco tcp syslog anomaly</a> +page. <li>$InputTCPServerRun <port><br> Starts a TCP server on selected port</li> <li><ul><li>$InputTCPMaxSessions <number></li></ul> Sets the maximum number of sessions supported</li><li>$InputTCPServerStreamDriverMode <number><br> -Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specifc.</li><li>$InputTCPServerStreamDriverAuthMode <mode-string><br> +Sets the driver mode for the currently selected <a href="netstream.html">network stream driver</a>. <number> is driver specifc.</li> +<li>$InputTCPServerInputName <name><br> +Sets a name for the inputname property. If no name is set "imtcp" is used by default. Setting a +name is not strictly necessary, but can be useful to apply filtering based on which input +the message was received from. +<li>$InputTCPServerStreamDriverAuthMode <mode-string><br> Sets the authentication mode for the currently selected <a href="netstream.html">network stream driver</a>. <mode-string> is driver specifc.</li><li>$InputTCPServerStreamDriverPermittedPeer <id-string><br> Sets permitted peer IDs. Only these peers are able to connect to the listener. <id-string> semantics depend on the currently selected @@ -31,7 +59,6 @@ AuthMode and <a href="netstream.html">network stream driver</a>. Permitted <b>Caveats/Known Bugs:</b> <ul> <li>module always binds to all interfaces</li> -<li>only a single listener can be bound</li> <li>can not be loaded together with <a href="imgssapi.html">imgssapi</a> (which includes the functionality of imtcp)</li> </ul> diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 77491992..472470a0 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -4,6 +4,8 @@ <title>Unix Socket Input</title> </head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>Unix Socket Input</h1> <p><b>Module Name: imuxsock</b></p> <p><b>Author: </b>Rainer Gerhards diff --git a/doc/log_rotation_fix_size.html b/doc/log_rotation_fix_size.html index 0b9a3b2e..190b24cb 100644 --- a/doc/log_rotation_fix_size.html +++ b/doc/log_rotation_fix_size.html @@ -3,6 +3,8 @@ <meta name="KEYWORDS" content="log rotation, howto, guide, fixed-size log"> </head> <body> +<a href="rsyslog_conf_output.html">back</a> + <h1>Log rotation with rsyslog</h1> <P><small><i>Written by Michael Meckelein</i></small></P> @@ -54,6 +56,14 @@ file and fill it up with new logs. So the latest logs are always in log_roatatio <p>With this approach two files for logging are used, each with a maximum size of 50 MB. So we can say we have successfully configured a log rotation which satisfies our requirement. We keep the logs at a fixed-size level of100 MB.</p> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> </body> </html> diff --git a/doc/manual.html b/doc/manual.html index 42ee5a8d..96497300 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,7 +16,10 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also <a href="professional_support.html">professional rsyslog support</a> available directly from the source!</p> -<p><b>This documentation is for version 3.22.1 (v3-stable branch) of rsyslog.</b> +<p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> +to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the +project goals.</p> +<p><b>This documentation is for version 5.1.2 (devel branch) of rsyslog.</b> Visit the <i> <a href="http://www.rsyslog.com/doc-status.html">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might @@ -33,13 +36,13 @@ the links below for the</b><br></p><ul> <li><a href="troubleshoot.html">troubleshooting rsyslog problems</a></li> <li><a href="rsyslog_conf.html">configuration file syntax (rsyslog.conf)</a></li> -<li> <a href="property_replacer.html">property replacer, an important core component</a></li> <li><a href="http://www.rsyslog.com/tool-regex">a regular expression checker/generator tool for rsyslog</a></li> +<li> <a href="property_replacer.html">property replacer, an important core component</a></li> <li>a commented <a href="sample.conf.html">sample rsyslog.conf</a> </li> <li><a href="bugs.html">rsyslog bug list</a></li> <li><a href="rsyslog_packages.html"> rsyslog packages</a></li> <li><a href="generic_design.html">backgrounder on -generic syslog application design</a><!-- not good as it currently is ;) <li><a href="contributors.html">contributor "Hall of Fame"</a>--></li> +generic syslog application design</a></li> <li><a href="modules.html">description of rsyslog modules</a></li> <li><a href="man_rsyslogd.html">rsyslogd man page</a> (heavily outdated)</li> </ul> @@ -49,6 +52,7 @@ generic syslog application design</a><!-- not good as it currently is ;) <li><a <li><a href="build_from_repo.html">obtaining rsyslog from the source repository</a></li> <li><a href="ipv6.html">rsyslog and IPv6</a> (which is fully supported)</li> <li><a href="rsyslog_secure_tls.html">native TLS encryption for syslog</a></li> +<li><a href="multi_ruleset.html">using multiple rule sets in rsyslog</a></li> <li><a href="rsyslog_stunnel.html">ssl-encrypting syslog with stunnel</a></li> <li><a href="rsyslog_mysql.html">writing syslog messages to MySQL (and other databases as well)</a></li> <li><a href="rsyslog_high_database_rate.html">writing massive amounts of syslog messages to a database</a></li> @@ -61,7 +65,11 @@ the syslog priority (severity and facility) to the log file</a></li> syslog sender over NAT</a> (online only)</li> <li><a href="gssapi.html">an overview and howto of rsyslog gssapi support</a></li> <li><a href="debug.html">debug support in rsyslog</a></li> -<li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li> +<li>Developer Documentation + <ul> + <li><a href="dev_oplugins.html">writing rsyslog output plugins</a></li> + <li><a href="dev_queue.html">the rsyslog message queue object (developer's view)</a></li> + </ul></li> </ul> <p>Our <a href="history.html">rsyslog history</a> page is for you if you would like to learn a little more diff --git a/doc/modules.html b/doc/modules.html index 92887508..4eae6db3 100644 --- a/doc/modules.html +++ b/doc/modules.html @@ -4,9 +4,8 @@ </head> <body> <h1>About rsyslog Modules</h1> - <P><small><i>Written by - <a href="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer - Gerhards</a> (2007-07-28)</i></small></P> +<P><small><i>Written by +<a href="http://www.adiscon.com/en/people/rainer-gerhards.php">Rainer Gerhards</a> (2007-07-28)</i></small></P> <p><font color="#FF0000"><b>This document is incomplete. The module interface is also quite incomplete and under development. Do not currently use it!</b></font> You may want to visit <a href="http://rgerhards.blogspot.com/">Rainer's blog</a> diff --git a/doc/multi_ruleset.html b/doc/multi_ruleset.html new file mode 100644 index 00000000..8d8c614f --- /dev/null +++ b/doc/multi_ruleset.html @@ -0,0 +1,275 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<title>Multiple Rulesets in rsyslog</title></head> +<body> +<h1>Multiple Rulesets in rsyslog</h1> +<p>Starting with version 4.5.0 and 5.1.1, <a href="http://www.rsyslog.com">rsyslog</a> supports +multiple rulesets within a single configuration. +This is especially useful for routing the recpetion of remote messages to a set of specific rules. +Note that the input module must support binding to non-standard rulesets, so the functionality +may not be available with all inputs. +<p>In this document, I am using <a href="imtcp.html">imtcp</a>, an input module +that supports binding to non-standard rulesets since rsyslog started to support them. +<h2>What is a Ruleset?</h2> +If you have worked with (r)syslog.conf, you know that it is made up of what I call rules (others +tend to call them selectors, a sysklogd term). Each rule consist of a filter and one or more +actions to be carried out when the filter evaluates to true. A filter may be as simple as a +traditional +syslog priority based filter (like "*.*" or "mail.info" or a as complex as a +script-like expression. Details on that are covered in the config file documentation. After the +filter come action specifiers, and an action is something that does something to a message, e.g. +write it to a file or forward it to a remote logging server. + +<p>A traditional configuration file is made up of one or more of these rules. When a new +message arrives, its processing starts with the first rule (in order of appearance in +rsyslog.conf) and continues for each rule until either all rules have been processed or +a so-called "e;discard" action happens, in which case processing stops and the +message is thrown away (what also happens after the last rule has been processed). + +<p>The <b>multi-ruleset</b> support now permits to specify more than one such rule sequence. +You can think of a traditional config file just as a single default rule set, which is +automatically bound to each of the inputs. This is even what actually happens. When +rsyslog.conf is processed, the config file parser looks for the directive + +<pre>$RuleSet <name> +</pre> + +<p>Where name is any name the user likes (but must not start with "RSYSLOG_", which +is the name space reserved for rsyslog use). If it finds this directive, it begins a new +rule set (if the name was not yet know) or switches to an already-existing one (if the name +was known). All rules defined between this $RuleSet directive and the next one are appended +to the named ruleset. Note that the reserved name "RSYSLOG_DefaultRuleset" is used to +specify rsyslogd's default ruleset. You can use that name whereever you can use a ruleset name, +including when binding an input to it. + +<p>Inside a ruleset, messages are processed as described above: they start with the first rule +and rules are processed in the order of appearance of the configuration file until either +there are no more rules or the discard action is executed. Note that with multiple rulesets +no longer <b>all</b> rsyslog.conf rules are executed but <b>only</b> those that are +contained within the specific ruleset. + +<p>Inputs must explicitely bind to rulesets. If they don't do, the default ruleset is bound. + +<p>This brings up the next question: + +<h2>What does "To bind to a Ruleset" mean?</h2> +<p>This term is used in the same sense as "to bind an IP address to an interface": +it means that a specific input, or part of an input (like a tcp listener) will use a specific +ruleset to "pass its messages to". So when a new message arrives, it will be processed +via the bound ruleset. Rule from all other rulesets are irrelevant and will never be processed. +<p>This makes multiple rulesets very handy to process local and remote message via +seperate means: bind the respective receivers to different rule sets, and you do not need +to seperate the messages by any other method. + +<p>Binding to rulesets is input-specifc. For imtcp, this is done via the + +<pre>$InputTCPServerBindRuleset <name> +</pre> + +directive. Note that "name"e; must be the name of a ruleset that is already defined +at the time the bind directive is given. There are many ways to make sure this happens, but +I personally think that it is best to define all rule sets at the top of rsyslog.conf and +define the inputs at the bottom. This kind of reverses the traditional recommended ordering, but +seems to be a really useful and straightforward way of doing things. +<h2>Can I use a different Ruleset as the default?</h2> +<p>This is possible by using the + +<pre>$DefaultRuleset <name> +</pre> + +Directive. Please note, however, that this directive is actually global: that is, it does not +modify the ruleset to which the next input is bound but rather provides a system-wide +default rule set for those inputs that did not explicitly bind to one. As such, the directive +can not be used as a work-around to bind inputs to non-default rulesets that do not support +ruleset binding. +<h2>Examples</h2> +<h3>Split local and remote logging</h3> +<p>Let's say you have a pretty standard system that logs its local messages to the usual +bunch of files that are specified in the default rsyslog.conf. As an example, your rsyslog.conf +might look like this: + +<pre> +# ... module loading ... +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +</pre> + +<p>Now, you want to add receive messages from a remote system and log these to +a special file, but you do not want to have these messages written to the files +specified above. The traditional approach is to add a rule in front of all others that +filters on the message, processes it and then discards it: + +<pre> +# ... module loading ... +# process remote messages +:fromhost-ip, isequal, "192.0.2.1" /var/log/remotefile +& ~ +# only messages not from 192.0.21 make it past this point + +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +</pre> + +<p>Note the tilde character, which is the discard action!. Also note that we assume that +192.0.2.1 is the sole remote sender (to keep it simple). + +<p>With multiple rulesets, we can simply define a dedicated ruleset for the remote reception +case and bind it to the receiver. This may be written as follows: + +<pre> +# ... module loading ... +# process remote messages +# define new ruleset and add rules to it: +$RuleSet remote +*.* /var/log/remotefile +# only messages not from 192.0.21 make it past this point + +# bind ruleset to tcp listener +$InputTCPServerBindRuleset remote +# and activate it: +$InputTCPServerRun 10514 + +# switch back to the default ruleset: +$RuleSet RSYSLOG_DefaultRuleset +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +</pre> + +<p>Here, we need to switch back to the default ruleset after we have defined our custom +one. This is why I recommend a different ordering, which I find more intuitive. The sample +below has it, and it leads to the same results: + +<pre> +# ... module loading ... +# at first, this is a copy of the unmodified rsyslog.conf +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +# end of the "regular" rsyslog.conf. Now come the new definitions: + +# process remote messages +# define new ruleset and add rules to it: +$RuleSet remote +*.* /var/log/remotefile + +# bind ruleset to tcp listener +$InputTCPServerBindRuleset remote +# and activate it: +$InputTCPServerRun 10514 +</pre> + +<p>Here, we do not switch back to the default ruleset, because this is not needed as it is +completely defined when we begin the "remote" ruleset. + +<p>Now look at the examples and compare them to the single-ruleset solution. You will notice +that we do <b>not</b> need a real filter in the multi-ruleset case: we can simply use +"*.*" as all messages now means all messages that are being processed by this +rule set and all of them come in via the TCP receiver! This is what makes using multiple +rulesets so much easier. + +<h3>Split local and remote logging for three different ports</h3> +<p>This example is almost like the first one, but it extends it a little bit. While it is +very similar, I hope it is different enough to provide a useful example why you may want +to have more than two rulesets. + +<p>Again, we would like to use the "regular" log files for local logging, only. But +this time we set up three syslog/tcp listeners, each one listening to a different +port (in this example 10514, 10515, and 10516). Logs received from these receivers shall go into +different files. Also, logs received from 10516 (and only from that port!) with +"mail.*" priority, shall be written into a specif file and <b>not</b> be +written to 10516's general log file. + +<p>This is the config: + +<pre> +# ... module loading ... +# at first, this is a copy of the unmodified rsyslog.conf +# The authpriv file has restricted access. +authpriv.* /var/log/secure +# Log all the mail messages in one place. +mail.* /var/log/maillog +# Log cron stuff +cron.* /var/log/cron +# Everybody gets emergency messages +*.emerg * +... more ... +# end of the "regular" rsyslog.conf. Now come the new definitions: + +# process remote messages + +#define rulesets first +$RuleSet remote10514 +*.* /var/log/remote10514 + +$RuleSet remote10515 +*.* /var/log/remote10515 + +$RuleSet remote10516 +mail.* /var/log/mail10516 +& ~ +# note that the discard-action will prevent this messag from +# being written to the remote10516 file - as usual... +*.* /var/log/remote10516 + +# and now define listners bound to the relevant ruleset +$InputTCPServerBindRuleset remote10514 +$InputTCPServerRun 10514 + +$InputTCPServerBindRuleset remote10515 +$InputTCPServerRun 10515 + +$InputTCPServerBindRuleset remote10516 +$InputTCPServerRun 10516 +</pre> + +<p>Note that the "mail.*" rule inside the "remote10516"e; ruleset does +not affect processing inside any other rule set, including the default rule set. + + +<h2>Performance</h2> +<p>No rule processing can be faster than not processing a rule at all. As such, it is useful +for a high performance system to identify disjunct actions and try to split these off to +different rule sets. In the example section, we had a case where three different tcp listeners +need to write to three different files. This is a perfect example of where multiple rule sets +are easier to use and offer more performance. The performance is better simply because there +is no need to check the reception service - instead messages are automatically pushed to the +right rule set and can be processed by very simple rules (maybe even with +"*.*"-filters, the fastest ones available). + +<p>In the long term, multiple rule sets will probably lay the foundation for even better +optimizations. So it is not a bad idea to get aquainted with them. + +<p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/netstream.html b/doc/netstream.html index e7d54c12..cbfa12ae 100644 --- a/doc/netstream.html +++ b/doc/netstream.html @@ -3,6 +3,8 @@ </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h1>Network Stream Drivers</h1><p>Network stream drivers are a layer between various parts of rsyslogd (e.g. the imtcp module) and the transport layer. They provide sequenced delivery, authentication and @@ -18,4 +20,4 @@ Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> -</body></html>
\ No newline at end of file +</body></html> diff --git a/doc/omlibdbi.html b/doc/omlibdbi.html index 8ff74371..ec1d01b6 100644 --- a/doc/omlibdbi.html +++ b/doc/omlibdbi.html @@ -4,6 +4,8 @@ </head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>Generic Database Output Module (omlibdbi)</h1> <p><b>Module Name: omlibdbi</b></p> <p><b>Author: </b>Rainer Gerhards diff --git a/doc/ommail.html b/doc/ommail.html index c18cf3f8..0841dc9f 100644 --- a/doc/ommail.html +++ b/doc/ommail.html @@ -1,6 +1,6 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head><title>mail output module - sending syslog messages via mail</title> - +<a href="features.html">back</a> </head> <body> <h1>Mail Output Module (ommail)</h1> @@ -142,4 +142,5 @@ Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> + </body></html> diff --git a/doc/ommysql.html b/doc/ommysql.html index e81ce532..9b35b402 100644 --- a/doc/ommysql.html +++ b/doc/ommysql.html @@ -5,6 +5,8 @@ </head> <body> +<a href="rsyslog_conf_modules.html">back</a> + <h1>MySQL Database Output Module</h1> <p><b>Module Name: ommysql</b></p> <p><b>Author: </b>Michael Meckelein (Initial Author) / Rainer Gerhards diff --git a/doc/omoracle.html b/doc/omoracle.html new file mode 100644 index 00000000..cfcf277f --- /dev/null +++ b/doc/omoracle.html @@ -0,0 +1,200 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"> +<title>Oracle Database Output Module</title> +</head> + +<body> +<a href="rsyslog_conf_modules.html">rsyslog module reference</a> + +<h1>Oracle Database Output Module</h1> +<p><b>Module Name: omoracle</b></p> +<p><b>Author: </b>Luis Fernando Muñoz Mejías <Luis.Fernando.Munoz.Mejias@cern.ch></p> +<p><b>Available since: </b>: 4.3.0 +<p><b>Status: </b>: contributed module, not maitained by rsyslog core authors +<p><b>Description</b>:</p> +<p>This module provides native support for logging to Oracle databases. It offers +superior performance over the more generic <a href="omlibdbi.html">omlibdbi</a> module. +It also includes a number of enhancements, most importantly prepared statements and +batching, what provides a big performance improvements. +</p> +<p>Note that this module is maintained by its original author. If you need assistance with it, +it is suggested to post questions to the +<a href="http://lists.adiscon.net/mailman/listinfo/rsyslog">rsyslog mailing list</a>. +<p>From the header comments of this module: +<p><pre> + + This is an output module feeding directly to an Oracle + database. It uses Oracle Call Interface, a propietary module + provided by Oracle. + + Selector lines to be used are of this form: + + :omoracle:;TemplateName + + The module gets its configuration via rsyslog $... directives, + namely: + + $OmoracleDBUser: user name to log in on the database. + + $OmoracleDBPassword: password to log in on the database. + + $OmoracleDB: connection string (an Oracle easy connect or a db + name as specified by tnsnames.ora) + + $OmoracleBatchSize: Number of elements to send to the DB on each + transaction. + + $OmoracleStatement: Statement to be prepared and executed in + batches. Please note that Oracle's prepared statements have their + placeholders as ':identifier', and this module uses the colon to + guess how many placeholders there will be. + + All these directives are mandatory. The dbstring can be an Oracle + easystring or a DB name, as present in the tnsnames.ora file. + + The form of the template is just a list of strings you want + inserted to the DB, for instance: + + $template TestStmt,"%hostname%%msg%" + + Will provide the arguments to a statement like + + $OmoracleStatement \ + insert into foo(hostname,message)values(:host,:message) + + Also note that identifiers to placeholders are arbitrarry. You + need to define the properties on the template in the correct order + you want them passed to the statement! +</pre> +<p>Some additional documentation contributed by Ronny Egner: +<pre> +REQUIREMENTS: +-------------- + +- Oracle Instantclient 10g (NOT 11g) Base + Devel + (if you´re on 64-bit linux you should choose the 64-bit libs!) +- JDK 1.6 (not neccessary for oracle plugin but "make" didd not finsished successfully without it) + +- "oracle-instantclient-config" script + (seems to shipped with instantclient 10g Release 1 but i was unable to find it for 10g Release 2 so here it is) + + +====================== /usr/local/bin/oracle-instantclient-config ===================== +#!/bin/sh +# +# Oracle InstantClient SDK config file +# Jean-Christophe Duberga - Bordeaux 2 University +# + +# just adapt it to your environment +incdirs="-I/usr/include/oracle/10.2.0.4/client64" +libdirs="-L/usr/lib/oracle/10.2.0.4/client64/lib" + +usage="\ +Usage: oracle-instantclient-config [--prefix[=DIR]] [--exec-prefix[=DIR]] [--version] [--cflags] [--libs] [--static-libs]" + +if test $# -eq 0; then + echo "${usage}" 1>&2 + exit 1 +fi + +while test $# -gt 0; do + case "$1" in + -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;; + *) optarg= ;; + esac + + case $1 in + --prefix=*) + prefix=$optarg + if test $exec_prefix_set = no ; then + exec_prefix=$optarg + fi + ;; + --prefix) + echo $prefix + ;; + --exec-prefix=*) + exec_prefix=$optarg + exec_prefix_set=yes + ;; + --exec-prefix) + echo ${exec_prefix} + ;; + --version) + echo ${version} + ;; + --cflags) + echo ${incdirs} + ;; + --libs) + echo $libdirs -lclntsh -lnnz10 -locci -lociei -locijdbc10 + ;; + --static-libs) + echo "No static libs" 1>&2 + exit 1 + ;; + *) + echo "${usage}" 1>&2 + exit 1 + ;; + esac + shift +done + +=============== END ============== + + + + +COMPILING RSYSLOGD +------------------- + + +./configure --enable-oracle + + + + +RUNNING +------- + +- make sure rsyslogd is able to locate the oracle libs (either via LD_LIBRARY_PATH or /etc/ld.so.conf) +- set TNS_ADMIN to point to your tnsnames.ora +- create a tnsnames.ora and test you are able to connect to the database + +- create user in oracle as shown in the following example: + create user syslog identified by syslog default tablespace users quota unlimited on users; + grant create session to syslog; + create role syslog_role; + grant syslog_role to syslog; + grant create table to syslog_role; + grant create sequence to syslog_role; + +- create tables as needed + +- configure rsyslog as shown in the following example + $ModLoad omoracle + + $OmoracleDBUser syslog + $OmoracleDBPassword syslog + $OmoracleDB syslog + $OmoracleBatchSize 1 + $OmoracleBatchItemSize 4096 + + $OmoracleStatementTemplate OmoracleStatement + $template OmoracleStatement,"insert into foo(hostname,message) values (:host,:message)" + $template TestStmt,"%hostname%%msg%" + *.* :omoracle:;TestStmt + (you guess it: username = password = database = "syslog".... see $rsyslogd_source/plugins/omoracle/omoracle.c for me info) +</pre> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/omrelp.html b/doc/omrelp.html index d5437a70..b3132d78 100644 --- a/doc/omrelp.html +++ b/doc/omrelp.html @@ -4,6 +4,8 @@ </head> <body> +<a href="rsyslog_conf_modules.html">back to rsyslog module documentation</a> + <h1>RELP Output Module (omrelp)</h1> <p><b>Module Name: omrelp</b></p> <p><b>Author: </b>Rainer Gerhards diff --git a/doc/omsnmp.html b/doc/omsnmp.html index 31aaef24..b38a594f 100644 --- a/doc/omsnmp.html +++ b/doc/omsnmp.html @@ -4,6 +4,7 @@ <title>SNMP Output Module</title></head> <body> +<a href="rsyslog_conf_modules.html">back</a> <h1>SNMP Output Module</h1> <p><b>Module Name: omsnmp</b></p> diff --git a/doc/professional_support.html b/doc/professional_support.html index 2cb6c1e1..7724ede8 100644 --- a/doc/professional_support.html +++ b/doc/professional_support.html @@ -49,6 +49,30 @@ configuration file, but will place easy to read comments in the places where you need to put them in. The agreement is governed under German law. You may also purchase this service if you would like to have your own configuration file reviewed, e.g. for auditing purposes.</p> +<h3>Local Installation Support</h3> +<p>If you intend to install rsyslog on your system but would like +to do so with minimal effort and according to your specification, you +can ask us to perform the installation for you. You get a perfect +installation, exactly like you needed, but without a need to +touch the system. This is a perfect choice for the busy administrator! +<p>In order to perform this work, we just need ssh access to your +system and the proper permissions. +<p>We charge a low one-time fee for this service. For details, please +contact <a href="mailto:info@adiscon.com">info@adiscon.com</a>. +<h3>Local Installation Maintenance</h3> +<p>If you used our services to set up the system, why not keep it +running perfectly with maintenance support? Under this contract, we +assure you run a recent build that does not interfere with your +environment and we even carry out change requests you may have. So this +is a hassle-free, everything cared about solution. +<p>Again, all we need to have is ssh access and the proper permissions +to your machine. Of course, work will only be carried out when you +expect us to do so. You are always in control of what happens. This +is a perfect outsourcing solution for those who would like to run +a great logging system but can not afford the time to keep it +in perfect shape! +<p>We charge a low monthly fee for this service. For details, please +contact <a href="mailto:info@adiscon.com">info@adiscon.com</a>. <h3>Custom Development</h3> <p>Do you need an exotic feature that otherwise would not be implemented? Do you need something really quick, quicker than it is available via @@ -85,4 +109,4 @@ Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 3 or higher.</font></p> -</body></html>
\ No newline at end of file +</body></html> diff --git a/doc/property_replacer.html b/doc/property_replacer.html index c2a0c0d2..7b604ea0 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -1,6 +1,7 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head><title>The Rsyslogd Property Replacer</title></head> <body> +<a href="rsyslog_conf_templates.html">back</a> <h1>The Property Replacer</h1> <p><b>The property replacer is a core component in rsyslogd's output system.</b> A syslog message has a number of @@ -29,10 +30,6 @@ Currently supported are:</p> socket. Should be useful for debugging.</td> </tr> <tr> -<td><b>uxtradmsg</b></td> -<td>will disappear soon - do NOT use!</td> -</tr> -<tr> <td><b>hostname</b></td> <td>hostname from the message</td> </tr> @@ -228,7 +225,7 @@ sequence with a regular expression is: "%msg:R:.*Sev:. \(.*\) \[.*--end%"</p> <p>It is possible to specify some parametes after the "R". These are comma-separated. They are: -<p>R,<regexp-type>,<submatch>,<nomatch>,<match-number> +<p>R,<regexp-type>,<submatch>,<<a href="rsyslog_conf_nomatch.html">nomatch</a>>,<match-number> <p>regexp-type is either "BRE" for Posix basic regular expressions or "ERE" for extended ones. The string must be given in upper case. The default is "BRE" to be consistent with earlier versions of rsyslog that @@ -240,13 +237,8 @@ that the first match is number 0, the second 1 and so on. Up to 10 matches (up to number 9) are supported. Please note that it would be more natural to have the match-number in front of submatch, but this would break backward-compatibility. So the match-number must be specified after "nomatch". -<p>nomatch is either "DFLT", "BLANK", ZERO or "FIELD" (all upper case!). It tells -what to use if no match is found. With "DFLT", the strig "**NO MATCH**" is -used. This was the only supported value up to rsyslog 3.19.5. With "BLANK" -a blank text is used (""). With "ZERO", "0" is used. -Finally, "FIELD" uses the full property text -instead of the expression. Some folks have requested that, so it seems -to be useful. +<p><a href="rsyslog_conf_nomatch.html">nomatch</a> specifies what should +be used in case no match is found. <p>The following is a sample of an ERE expression that takes the first submatch from the message string and replaces the expression with the full field if no match is found: @@ -318,6 +310,18 @@ case-insensitive. Currently, the following options are defined: <td>convert property text to uppercase only</td> </tr> <tr> +<td valign="top"><b>csv</b></td> +<td>formats the resulting field (after all modifications) in CSV format +as specified in <a href="http://www.ietf.org/rfc/rfc4180.txt">RFC 4180</a>. +Rsyslog will always use double quotes. Note that in order to have full CSV-formatted +text, you need to define a proper template. An example is this one: +<br>$template csvline,"%syslogtag:::csv%,%msg:::csv%" +<br>Most importantly, you need to provide the commas between the fields +inside the template. +<br><i>This feature was introduced in rsyslog 4.1.6.</i> +</td> +</tr> +<tr> <td><b>drop-last-lf</b></td> <td>The last LF in the message (if any), is dropped. Especially useful for PIX.</td> @@ -410,4 +414,13 @@ to record severity and facility of a message)</li> <li><a href="rsyslog_conf.html">Configuration file syntax</a>, this is where you actually use the property replacer.</li> </ul> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> + </body></html> diff --git a/doc/queue_analogy_tv.png b/doc/queue_analogy_tv.png Binary files differnew file mode 100644 index 00000000..fedcb558 --- /dev/null +++ b/doc/queue_analogy_tv.png diff --git a/doc/queue_msg_state.dot b/doc/queue_msg_state.dot new file mode 100644 index 00000000..bfef2657 --- /dev/null +++ b/doc/queue_msg_state.dot @@ -0,0 +1,25 @@ +// This file is part of rsyslog. +// +// rsyslog message state in queue processing +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + rankdir=LR + + prod [label="producer" style="dotted" shape="box"] + que [label="queued"] + deq [label="dequeued"] + del [label="deleted"] + + prod -> que [label="qEnq()" style="dotted"] + que -> deq [label="qDeq()"] + deq -> del [label="qDel()"] + deq -> que [label="fatal failure\n& restart"] + + //{rank=same; del apf pdn } +} diff --git a/doc/queue_msg_state.jpeg b/doc/queue_msg_state.jpeg Binary files differnew file mode 100644 index 00000000..a215f000 --- /dev/null +++ b/doc/queue_msg_state.jpeg diff --git a/doc/queues.html b/doc/queues.html index a2074d36..75b70fbf 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -1,14 +1,21 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head> -<meta http-equiv="Content-Language" content="de"> <title>Understanding rsyslog queues</title></head> <body> +<a href="rsyslog_conf_global.html">back</a> <h1>Understanding rsyslog Queues</h1> <p>Rsyslog uses queues whenever two activities need to be loosely coupled. With a queue, one part of the system "produces" something while another part "consumes" this something. The "something" is most often syslog messages, but queues may also be used for other purposes.</p> +<p>This document provides a good insight into technical details, operation modes +and implications. In addition to it, an +<a href="queues_analogy.html">rsyslog queue concepts overview</a> document +exists which tries to explain queues with the help of some analogies. This may +probably be a better place to start reading about queues. I assume that once you +have understood that document, the material here will be much easier to grasp +and look much more natural. <p>The most prominent example is the main message queue. Whenever rsyslog receives a message (e.g. locally, via UDP, TCP or in whatever else way), it places these messages into the main message queue. Later, it is dequeued by the @@ -16,6 +23,36 @@ rule processor, which then evaluates which actions are to be carried out. In front of each action, there is also a queue, which potentially de-couples the filter processing from the actual action (e.g. writing to file, database or forwarding to another host).</p> +<h1>Where are Queues Used?</h1> +<p>Currently, queues are used for the main message queue and for the +actions.</p> +<p>There is a single main message queue inside rsyslog. Each input module +delivers messages to it. The main message queue worker filters messages based on +rules specified in rsyslog.conf and dispatches them to the individual action +queues. Once a message is in an action queue, it is deleted from the main +message queue.</p> +<p>There are multiple action queues, one for each configured action. By default, +these queues operate in direct (non-queueing) mode. Action queues are fully +configurable and thus can be changed to whatever is best for the given use case.</p> +<p>Future versions of rsyslog will most probably utilize queues at other places, +too.</p> +<p> Wherever "<i><object></i>" is used in the config file +statements, substitute "<i><object></i>" with either "MainMsg" or "Action". The +former will set main message queue +parameters, the later parameters for the next action that will be +created. Action queue parameters can not be modified once the action has been +specified. For example, to tell the main message queue to save its content on +shutdown, use <i>$MainMsgQueueSaveOnShutdown on</i>".</p> +<p>If the same parameter is specified multiple times before a queue is created, +the last one specified takes precedence. The main message queue is created after +parsing the config file and all of its potential includes. An action queue is +created each time an action selector is specified. Action queue parameters are +reset to default after an action queue has been created (to provide a clean +environment for the next action).</p> +<p>Not all queues necessarily support the full set of queue configuration +parameters, because not all are applicable. For example, in current output +module design, actions do not support multi-threading. Consequently, the number +of worker threads is fixed to one for action queues and can not be changed.</p> <h1>Queue Modes</h1> <p>Rsyslog supports different queue modes, some with submodes. Each of them has specific advantages and disadvantages. Selecting the right queue mode is quite @@ -78,7 +115,11 @@ isolation. This is currently selected by specifying different <i>$WorkDirectory< config directives before the queue creation statement.</p> <p>To create a disk queue, use the "<i>$<object>QueueType Disk</i>" config directive. Checkpoint intervals can be specified via "<i>$<object>QueueCheckpointInterval</i>", -with 0 meaning no checkpoints. </p> +with 0 meaning no checkpoints. Note that disk-based queues can be made very reliable +by issuing a (f)sync after each write operation. Starting with version 4.3.2, this can +be requested via "<i><object>QueueSyncQueueFiles on/off</i> with the +default being off. Activating this option has a performance penalty, so it should +not be turned on without reason.</p> <h2>In-Memory Queues</h2> <p>In-memory queue mode is what most people have on their mind when they think about computing queues. Here, the enqueued data elements are held in memory. @@ -113,8 +154,7 @@ only memory if in use. A FixedArray queue may have a too large static memory footprint in such cases.</p> <p><b>In general, it is advised to use LinkedList mode if in doubt</b>. The processing overhead compared to FixedArray is low and may be -<span style="font-size: 12pt; line-height: 115%; font-family: 'Times New Roman',serif;" lang="EN-US"> -outweigh </span>by the reduction in memory use. Paging in most-often-unused +outweigh by the reduction in memory use. Paging in most-often-unused pointer array pages can be much slower than dynamically allocating them.</p> <p>To create an in-memory queue, use the "<i>$<object>QueueType LinkedList</i>" or "<i>$<object>QueueType FixedArray</i>" config directive.</p> @@ -219,11 +259,12 @@ parall. Thus, the upper limit ca be set via "<i>$<object>QueueWorkerThread If it, for example, is set to four, no more than four workers will ever be started, no matter how many elements are enqueued. </p> <p>Worker threads that have been started are kept running until an inactivity -timeout happens. The timeout can be set via "<i>$<object>QueueWorkerTimeoutShutdown</i>" +timeout happens. The timeout can be set via "<i>$<object>QueueWorkerTimeoutThreadShutdown</i>" and is specified in milliseconds. If you do not like to keep the workers running, simply set it to 0, which means immediate timeout and thus immediate shutdown. But consider that creating threads involves some overhead, and this is -why we keep them running.</p> +why we keep them running. If you would like to never shutdown any worker +threads, specify -1 for this parameter.</p> <h2>Discarding Messages</h2> <p>If the queue reaches the so called "discard watermark" (a number of queued elements), less important messages can automatically be discarded. This is in an @@ -258,19 +299,15 @@ unavoidable and you prefer to discard less important messages first.</p> disk space, it is finally full. If so, rsyslogd throttles the data element submitter. If that, for example, is a reliable input (TCP, local log socket), that will slow down the message originator which is a good -<span style="font-size: 12pt; line-height: 115%; font-family: 'Times New Roman',serif;" lang="EN-US"> -resolution </span>for this scenario.</p> -<p>During -<span style="font-size: 12pt; line-height: 115%; font-family: 'Times New Roman',serif;" lang="EN-US"> -throtteling</span>, a disk-assisted queue continues to write to disk and +resolution for this scenario.</p> +<p>During throtteling, a disk-assisted queue continues to write to disk and messages are also discarded based on severity as well as regular dequeuing and processing continues. So chances are good the situation will be resolved by simply throttling. Note, though, that throtteling is highly undesirable for unreliable sources, like UDP message reception. So it is not a good thing to run into throtteling mode at all.</p> <p>We can not hold processing -<span style="font-size: 12pt; line-height: 115%; font-family: 'Times New Roman',serif;" lang="EN-US"> -infinitely</span>, not even when throtteling. For example, throtteling the local +infinitely, not even when throtteling. For example, throtteling the local log socket too long would cause the system at whole come to a standstill. To prevent this, rsyslogd times out after a configured period ("<i>$<object>QueueTimeoutEnqueue</i>", specified in milliseconds) if no space becomes available. As a last resort, it @@ -299,10 +336,36 @@ in this regard - it was just not requested so far. So if you need more fine-grained control, let us know and we'll probably implement it. There are two configuration directives, both should be used together or results are unpredictable:" <i>$<object>QueueDequeueTimeBegin <hour></i>" and "<i>$<object>QueueDequeueTimeEnd <hour></i>". The hour parameter must be specified in 24-hour format (so 10pm is 22). A use case for this parameter can be found in the <a href="http://wiki.rsyslog.com/index.php/OffPeakHours">rsyslog wiki</a>. </p> +<h2>Performance</h2> +<p>The locking involved with maintaining the queue has a potentially large +performance impact. How large this is, and if it exists at all, depends much on +the configuration and actual use case. However, the queue is able to work on +so-called "batches" when dequeueing data elements. With batches, +multiple data elements are dequeued at once (with a single locking call). +The queue dequeues all available elements up to a configured upper +limit (<i><object>DequeueBatchSize <number></i>). It is important +to note that the actual upper limit is dictated by availability. The queue engine +will never wait for a batch to fill. So even if a high upper limit is configured, +batches may consist of fewer elements, even just one, if there are no more elements +waiting in the queue. +<p>Batching +can improve performance considerably. Note, however, that it affects the +order in which messages are passed to the queue worker threads, as each worker +now receive as batch of messages. Also, the larger the batch size and the higher +the maximum number of permitted worker threads, the more main memory is needed. +For a busy server, large batch sizes (around 1,000 or even more elements) may be useful. +Please note that with batching, the main memory must hold BatchSize * NumOfWorkers +objects in memory (worst-case scenario), even if running in disk-only mode. So if you +use the default 5 workers at the main message queue and set the batch size to 1,000, you need +to be prepared that the main message queue holds up to 5,000 messages in main memory +<b>in addition</b> to the configured queue size limits! +<p>The queue object's default maximum batch size +is eight, but there exists different defaults for the actual parts of +rsyslog processing that utilize queues. So you need to check these object's +defaults. <h2>Terminating Queues</h2> <p>Terminating a process sounds easy, but can be complex. -<span style="font-size: 12pt; line-height: 115%; font-family: 'Times New Roman',serif;" lang="EN-US"> -Terminating </span>a running queue is in fact the most complex operation a queue +Terminating a running queue is in fact the most complex operation a queue object can perform. You don't see that from a user's point of view, but its quite hard work for the developer to do everything in the right order.</p> <p>The complexity arises when the queue has still data enqueued when it @@ -323,38 +386,13 @@ it terminates. This includes data elements there were begun being processed by workers that needed to be cancelled due to too-long processing. For a large queue, this operation may be lengthy. No timeout applies to a required shutdown save.</p> -<h1>Where are Queues Used?</h1> -<p> Currently, queues are used for the main message queue and for the -actions.</p> -<p>There is a single main message queue inside rsyslog. Each input module -delivers messages to it. The main message queue worker filters messages based on -rules specified in rsyslog.conf and dispatches them to the individual action -queues. Once a message is in an action queue, it is deleted from the main -message queue.</p> -<p>There are multiple action queues, one for each configured action. By default, -these queues operate in direct (non-queueing) mode. Action queues are fully -configurable and thus can be changed to whatever is best for the given use case.</p> -<p>Future versions of rsyslog will most probably utilize queues at other places, -too.</p> -<p> -<span style="font-size: 12pt; line-height: 115%; font-family: 'Times New Roman',serif;" lang="EN-US"> -Wherever </span>"<i><object></i>" was used above in the config file -statements, substitute "<i><object></i>" with either "MainMsg" or "Action". The -former will set main message queue -<span style="font-size: 12pt; line-height: 115%; font-family: 'Times New Roman',serif;" lang="EN-US"> -parameters</span>, the later parameters for the next action that will be -created. Action queue parameters can not be modified once the action has been -specified. For example, to tell the main message queue to save its content on -shutdown, use <i>$MainMsgQueueSaveOnShutdown on</i>".</p> -<p>If the same parameter is specified multiple times before a queue is created, -the last one specified takes precedence. The main message queue is created after -parsing the config file and all of its potential includes. An action queue is -created each time an action selector is specified. Action queue parameters are -reset to default after an action queue has been created (to provide a clean -environment for the next action).</p> -<p>Not all queues necessarily support the full set of queue configuration -parameters, because not all are applicable. For example, in current output -module design, actions do not support multi-threading. Consequently, the number -of worker threads is fixed to one for action queues and can not be changed.</p> +[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> -</body></html>
\ No newline at end of file +</body></html> diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html new file mode 100644 index 00000000..d7533ad5 --- /dev/null +++ b/doc/queues_analogy.html @@ -0,0 +1,259 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<title>turning lanes and rsyslog queues - an analogy</title></head> +<body> +<a href="rsyslog_conf_global.html">back</a> + +<h1>Turning Lanes and Rsyslog Queues - an Analogy</h1> +<p>If there is a single object absolutely vital to understanding the way +rsyslog works, this object is queues. Queues offer a variety of services, +including support for multithreading. While there is elaborate in-depth +documentation on the ins and outs of <a href="queues.html">rsyslog queues</a>, +some of the concepts are hard to grasp even for experienced people. I think this +is because rsyslog uses a very high layer of abstraction which includes things +that look quite unnatural, like queues that do <b>not</b> actually queue... +<p>With this document, I take a different approach: I will not describe every specific +detail of queue operation but hope to be able to provide the core idea of how +queues are used in rsyslog by using an analogy. I will compare the rsyslog data flow +with real-life traffic flowing at an intersection. +<p>But first let's set the stage for the rsyslog part. The graphic below describes +the data flow inside rsyslog: +<p align="center"><img src="dataflow.png" alt="rsyslog data flow"> +<p>Note that there is a <a href="http://www.rsyslog.com/Article350.phtml">video tutorial</a> +available on the data flow. It is not perfect, but may aid in understanding this picture. +<p>For our needs, the important fact to know is that messages enter rsyslog on "the +left side" (for example, via UDP), are being preprocessed, put into the +so-called main queue, taken off that queue, be filtered and be placed into one or +several action queues (depending on filter results). They leave rsyslog on "the +right side" where output modules (like the file or database writer) consume them. +<p>So there are always <b>two</b> stages where a message (conceptually) is queued - first +in the main queue and later on in <i>n</i> action specific queues (with <i>n</i> being the number of +actions that the message in question needs to be processed by, what is being decided +by the "Filter Engine"). As such, a message will be in at least two queues +during its lifetime (with the exception of messages being discarded by the queue itself, +but for the purpose of this document, we will ignore that possibility). +<p>Also, it is vitally +important to understand that <b>each</b> action has a queue sitting in front of it. +If you have dug into the details of rsyslog configuration, you have probably seen that +a queue mode can be set for each action. And the default queue mode is the so-called +"direct mode", in which "the queue does not actually enqueue data". +That sounds silly, but is not. It is an important abstraction that helps keep the code clean. +<p>To understand this, we first need to look at who is the active component. In our data flow, +the active part always sits to the left of the object. For example, the "Preprocessor" +is being called by the inputs and calls itself into the main message queue. That is, the queue +receiver is called, it is passive. One might think that the "Parser & Filter Engine" +is an active component that actively pulls messages from the queue. This is wrong! Actually, +it is the queue that has a pool of worker threads, and these workers pull data from the queue +and then call the passively waiting Parser and Filter Engine with those messages. So the +main message queue is the active part, the Parser and Filter Engine is passive. +<p>Let's now try an analogy analogy for this part: Think about a TV show. The show is produced +in some TV studio, from there sent (actively) to a radio tower. The radio tower passively +receives from the studio and then actively sends out a signal, which is passively received +by your TV set. In our simplified view, we have the following picture: +<p align="center"><img src="queue_analogy_tv.png" alt="rsyslog queues and TV analogy"> +<p>The lower part of the picture lists the equivalent rsyslog entities, in an abstracted way. +Every queue has a producer (in the above sample the input) and a consumer (in the above sample the Parser +and Filter Engine). Their active and passive functions are equivalent to the TV entities +that are listed on top of the rsyslog entity. For example, a rsyslog consumer can never +actively initiate reception of a message in the same way a TV set cannot actively +"initiate" a TV show - both can only "handle" (display or process) +what is sent to them. +<p>Now let's look at the action queues: here, the active part, the producer, is the +Parser and Filter Engine. The passive part is the Action Processor. The later does any +processing that is necessary to call the output plugin, in particular it processes the template +to create the plugin calling parameters (either a string or vector of arguments). From the +action queue's point of view, Action Processor and Output form a single entity. Again, the +TV set analogy holds. The Output <b>does not</b> actively ask the queue for data, but +rather passively waits until the queue itself pushes some data to it. + +<p>Armed with this knowledge, we can now look at the way action queue modes work. My analogy here +is a junction, as shown below (note that the colors in the pictures below are <b>not</b> related to +the colors in the pictures above!): +<p align="center"><img src="direct_queue0.png"> +<p>This is a very simple real-life traffic case: one road joins another. We look at +traffic on the straight road, here shown by blue and green arrows. Traffic in the +opposing direction is shown in blue. Traffic flows without +any delays as long as nobody takes turns. To be more precise, if the opposing traffic takes +a (right) turn, traffic still continues to flow without delay. However, if a car in the red traffic +flow intends to do a (left, then) turn, the situation changes: +<p align="center"><img src="direct_queue1.png"> +<p>The turning car is represented by the green arrow. It cannot turn unless there is a gap +in the "blue traffic stream". And as this car blocks the roadway, the remaining +traffic (now shown in red, which should indicate the block condition), +must wait until the "green" car has made its turn. So +a queue will build up on that lane, waiting for the turn to be completed. +Note that in the examples below I do not care that much about the properties of the +opposing traffic. That is, because its structure is not really important for what I intend to +show. Think about the blue arrow as being a traffic stream that most of the time blocks +left-turners, but from time to time has a gap that is sufficiently large for a left-turn +to complete. +<p>Our road network designers know that this may be unfortunate, and for more important roads +and junctions, they came up with the concept of turning lanes: +<p align="center"><img src="direct_queue2.png"> +<p>Now, the car taking the turn can wait in a special area, the turning lane. As such, +the "straight" traffic is no longer blocked and can flow in parallel to the +turning lane (indicated by a now-green-again arrow). + +<p>However, the turning lane offers only finite space. So if too many cars intend to +take a left turn, and there is no gap in the "blue" traffic, we end up with +this well-known situation: +<p align="center"><img src="direct_queue3.png"> +<p>The turning lane is now filled up, resulting in a tailback of cars intending to +left turn on the main driving lane. The end result is that "straight" traffic +is again being blocked, just as in our initial problem case without the turning lane. +In essence, the turning lane has provided some relief, but only for a limited amount of +cars. Street system designers now try to weight cost vs. benefit and create (costly) +turning lanes that are sufficiently large to prevent traffic jams in most, but not all +cases. +<p><b>Now let's dig a bit into the mathematical properties of turning lanes.</b> We assume that +cars all have the same length. So, units of cars, the length is alsways one (which is nice, +as we don't need to care about that factor any longer ;)). A turning lane has finite capacity of +<i>n</i> cars. As long as the number of cars wanting to take a turn is less than or eqal +to <i>n</i>, "straigth traffic" is not blocked (or the other way round, traffic +is blocked if at least <i>n + 1</i> cars want to take a turn!). We can now find an optimal +value for <i>n</i>: it is a function of the probability that a car wants to turn +and the cost of the turning lane +(as well as the probability there is a gap in the "blue" traffic, but we ignore this +in our simple sample). +If we start from some finite upper bound of <i>n</i>, we can decrease +<i>n</i> to a point where it reaches zero. But let's first look at <i>n = 1</i>, in which case exactly +one car can wait on the turning lane. More than one car, and the rest of the traffic is blocked. +Our everyday logic indicates that this is actually the lowest boundary for <i>n</i>. +<p>In an abstract view, however, <i>n</i> can be zero and that works nicely. There still can be +<i>n</i> cars at any given time on the turning lane, it just happens that this means there can +be no car at all on it. And, as usual, if we have at least <i>n + 1</i> cars wanting to turn, +the main traffic flow is blocked. True, but <i>n + 1 = 0 + 1 = 1</i> so as soon as there is any +car wanting to take a turn, the main traffic flow is blocked (remember, in all cases, I assume +no sufficiently large gaps in the opposing traffic). +<p>This is the situation our everyday perception calls "road without turning lane". +In my math model, it is a "road with turning lane of size 0". The subtle difference +is important: my math model guarantees that, in an abstract sense, there always is a turning +lane, it may just be too short. But it exists, even though we don't see it. And now I can +claim that even in my small home village, all roads have turning lanes, which is rather +impressive, isn't it? ;) +<p><b>And now we finally have arrived at rsyslog's queues!</b> Rsyslog action queues exists for +all actions just like all roads in my village have turning lanes! And as in this real-life sample, +it may be hard to see the action queues for that reason. In rsyslog, the "direct" queue +mode is the equivalent to the 0-sized turning lane. And actions queues are the equivalent to turning +lanes in general, with our real-life <i>n</i> being the maximum queue size. +The main traffic line (which sometimes is blocked) is the equivalent to the +main message queue. And the periods without gaps in the opposing traffic are equivalent to +execution time of an action. In a rough sketch, the rsyslog main and action queues look like in the +following picture. +<p align="center"><img src="direct_queue_rsyslog.png"> +<p>We need to read this picture from right to left (otherwise I would need to redo all +the graphics ;)). In action 3, you see a 0-sized turning lane, aka an action queue in "direct" +mode. All other queues are run in non-direct modes, but with different sizes greater than 0. +<p>Let us first use our car analogy: +Assume we are in a car on the main lane that wants to take turn into the "action 4" +road. We pass action 1, where a number of cars wait in the turning lane and we pass +action 2, which has a slightly smaller, but still not filled up turning lane. So we pass that +without delay, too. Then we come to "action 3", which has no turning lane. Unfortunately, +the car in front of us wants to turn left into that road, so it blocks the main lane. So, this time +we need to wait. An observer standing on the sidewalk may see that while we need to wait, there are +still some cars in the "action 4" turning lane. As such, even though no new cars can +arrive on the main lane, cars still turn into the "action 4" lane. In other words, +an observer standing in "action 4" road is unable to see that traffic on the main lane +is blocked. +<p>Now on to rsyslog: Other than in the real-world traffic example, messages in rsyslog +can - at more or less the +same time - "take turns" into several roads at once. This is done by duplicating the message +if the road has a non-zero-sized "turning lane" - or in rsyslog terms a queue that is +running in any non-direct mode. If so, a deep copy of the message object is made, that placed into +the action queue and then the initial message proceeds on the "main lane". The action +queue then pushes the duplicates through action processing. This is also the reason why a +discard action inside a non-direct queue does not seem to have an effect. Actually, it discards the +copy that was just created, but the original message object continues to flow. +<p> +In action 1, we have some entries in the action queue, as we have in action 2 (where the queue is +slightly shorter). As we have seen, new messages pass action one and two almost instantaneously. +However, when a messages reaches action 3, its flow is blocked. Now, message processing must wait +for the action to complete. Processing flow in a direct mode queue is something like a U-turn: + +<p align="center"><img src="direct_queue_directq.png" alt="message processing in an rsyslog action queue in direct mode"> +<p>The message starts to execute the action and once this is done, processing flow continues. +In a real-life analogy, this may be the route of a delivery man who needs to drop a parcel +in a side street before he continues driving on the main route. As a side-note, think of what happens +with the rest of the delivery route, at least for today, if the delivery truck has a serious accident +in the side street. The rest of the parcels won't be delivered today, will they? This is exactly how the +discard action works. It drops the message object inside the action and thus the message will no +longer be available for further delivery - but as I said, only if the discard is done in a +direct mode queue (I am stressing this example because it often causes a lot of confusion). +<p>Back to the overall scenario. We have seen that messages need to wait for action 3 to +complete. Does this necessarily mean that at the same time no messages can be processed +in action 4? Well, it depends. As in the real-life scenario, action 4 will continue to +receive traffic as long as its action queue ("turn lane") is not drained. In +our drawing, it is not. So action 4 will be executed while messages still wait for action 3 +to be completed. +<p>Now look at the overall picture from a slightly different angle: +<p align="center"><img src="direct_queue_rsyslog2.png" alt="message processing in an rsyslog action queue in direct mode"> +<p>The number of all connected green and red arrows is four - one each for action 1, 2 and 3 +(this one is dotted as action 4 was a special case) and one for the "main lane" as +well as action 3 (this one contains the sole red arrow). <b>This number is the lower bound for +the number of threads in rsyslog's output system ("right-hand part" of the main message +queue)!</b> Each of the connected arrows is a continuous thread and each "turn lane" is +a place where processing is forked onto a new thread. Also, note that in action 3 the processing +is carried out on the main thread, but not in the non-direct queue modes. +<p>I have said this is "the lower bound for the number of threads...". This is with +good reason: the main queue may have more than one worker thread (individual action queues +currently do not support this, but could do in the future - there are good reasons for that, too +but exploring why would finally take us away from what we intend to see). Note that you +configure an upper bound for the number of main message queue worker threads. The actual number +varies depending on a lot of operational variables, most importantly the number of messages +inside the queue. The number <i>t_m</i> of actually running threads is within the integer-interval +[0,confLimit] (with confLimit being the operator configured limit, which defaults to 5). +Output plugins may have more than one thread created by themselves. It is quite unusual for an +output plugin to create such threads and so I assume we do not have any of these. +Then, the overall number of threads in rsyslog's filtering and output system is +<i>t_total = t_m + number of actions in non-direct modes</i>. Add the number of +inputs configured to that and you have the total number of threads running in rsyslog at +a given time (assuming again that inputs utilize only one thread per plugin, a not-so-safe +assumption). +<p>A quick side-note: I gave the lower bound for <i>t_m</i> as zero, which is somewhat in contrast +to what I wrote at the begin of the last paragraph. Zero is actually correct, because rsyslog +stops all worker threads when there is no work to do. This is also true for the action queues. +So the ultimate lower bound for a rsyslog output system without any work to carry out actually is zero. +But this bound will never be reached when there is continuous flow of activity. And, if you are +curios: if the number of workers is zero, the worker wakeup process is actually handled within the +threading context of the "left-hand-side" (or producer) of the queue. After being +started, the worker begins to play the active queue component again. All of this, of course, +can be overridden with configuration directives. +<p>When looking at the threading model, one can simply add n lanes to the main lane but otherwise +retain the traffic analogy. This is a very good description of the actual process (think what this +means to the "turning lanes"; hint: there still is only one per action!). +<p><b>Let's try to do a warp-up:</b> I have hopefully been able to show that in rsyslog, an action +queue "sits in front of" each output plugin. Messages are received and flow, from input +to output, over various stages and two level of queues to the outputs. Actions queues are always +present, but may not easily be visible when in direct mode (where no actual queuing takes place). +The "road junction with turning lane" analogy well describes the way - and intent - of the various +queue levels in rsyslog. +<p>On the output side, the queue is the active component, <b>not</b> the consumer. As such, the consumer +cannot ask the queue for anything (like n number of messages) but rather is activated by the queue +itself. As such, a queue somewhat resembles a "living thing" whereas the outputs are +just tools that this "living thing" uses. +<p><b>Note that I left out a couple of subtleties</b>, especially when it comes +to error handling and terminating +a queue (you hopefully have now at least a rough idea why I say "terminating <b>a queue</b>" +and not "terminating an action" - <i>who is the "living thing"?</i>). An action returns +a status to the queue, but it is the queue that ultimately decides which messages can finally be +considered processed and which not. Please note that the queue may even cancel an output right in +the middle of its action. This happens, if configured, if an output needs more than a configured +maximum processing time and is a guard condition to prevent slow outputs from deferring a rsyslog +restart for too long. Especially in this case re-queuing and cleanup is not trivial. Also, note that +I did not discuss disk-assisted queue modes. The basic rules apply, but there are some additional +constraints, especially in regard to the threading model. Transitioning between actual +disk-assisted mode and pure-in-memory-mode (which is done automatically when needed) is also far from +trivial and a real joy for an implementer to work on ;). +<p>If you have not done so before, it may be worth reading the +<a href="queues.html">rsyslog queue user's guide,</a> which most importantly lists all +the knobs you can turn to tweak queue operation. +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_actionexeconlywhenpreviousissuspended.html b/doc/rsconf1_actionexeconlywhenpreviousissuspended.html index d5cf8b14..1626b4ca 100644 --- a/doc/rsconf1_actionexeconlywhenpreviousissuspended.html +++ b/doc/rsconf1_actionexeconlywhenpreviousissuspended.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$ActionExecOnlyWhenPreviousIsSuspended</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> off</p> diff --git a/doc/rsconf1_actionresumeinterval.html b/doc/rsconf1_actionresumeinterval.html index a854a212..c0365470 100644 --- a/doc/rsconf1_actionresumeinterval.html +++ b/doc/rsconf1_actionresumeinterval.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$ActionResumeInterval</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> 30</p> @@ -27,4 +29,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_allowedsender.html b/doc/rsconf1_allowedsender.html index 4a980b89..ac39e268 100644 --- a/doc/rsconf1_allowedsender.html +++ b/doc/rsconf1_allowedsender.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$AllowedSender</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> all allowed</p> diff --git a/doc/rsconf1_controlcharacterescapeprefix.html b/doc/rsconf1_controlcharacterescapeprefix.html index 6dab1e2e..45cd9230 100644 --- a/doc/rsconf1_controlcharacterescapeprefix.html +++ b/doc/rsconf1_controlcharacterescapeprefix.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$ControlCharacterEscapePrefix</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> \</p> diff --git a/doc/rsconf1_debugprintcfsyslinehandlerlist.html b/doc/rsconf1_debugprintcfsyslinehandlerlist.html index 1aad7552..e158de43 100644 --- a/doc/rsconf1_debugprintcfsyslinehandlerlist.html +++ b/doc/rsconf1_debugprintcfsyslinehandlerlist.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$DebugPrintCFSyslineHandlerList</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> on</p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_debugprintmodulelist.html b/doc/rsconf1_debugprintmodulelist.html index 4d8e9bff..f25663fb 100644 --- a/doc/rsconf1_debugprintmodulelist.html +++ b/doc/rsconf1_debugprintmodulelist.html @@ -3,6 +3,7 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> <h2>$DebugPrintModuleList</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> on</p> @@ -19,4 +20,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_debugprinttemplatelist.html b/doc/rsconf1_debugprinttemplatelist.html index 243530e1..b5f1f28f 100644 --- a/doc/rsconf1_debugprinttemplatelist.html +++ b/doc/rsconf1_debugprinttemplatelist.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$DebugPrintTemplateList</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> on</p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_dircreatemode.html b/doc/rsconf1_dircreatemode.html index 66a35e18..9a9c61eb 100644 --- a/doc/rsconf1_dircreatemode.html +++ b/doc/rsconf1_dircreatemode.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$DirCreateMode</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> 0644</p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_dirgroup.html b/doc/rsconf1_dirgroup.html index 868e5ecd..de070126 100644 --- a/doc/rsconf1_dirgroup.html +++ b/doc/rsconf1_dirgroup.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$DirGroup</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_dirowner.html b/doc/rsconf1_dirowner.html index e85a5122..da8e252d 100644 --- a/doc/rsconf1_dirowner.html +++ b/doc/rsconf1_dirowner.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$DirOwner</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html b/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html index e0a53ae6..95027a70 100644 --- a/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html +++ b/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$DropMsgsWithMaliciousDnsPTRRecords</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> off</p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_droptrailinglfonreception.html b/doc/rsconf1_droptrailinglfonreception.html index 1e3aa8af..fb59b871 100644 --- a/doc/rsconf1_droptrailinglfonreception.html +++ b/doc/rsconf1_droptrailinglfonreception.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$DropTrailingLFOnReception</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> on</p> diff --git a/doc/rsconf1_dynafilecachesize.html b/doc/rsconf1_dynafilecachesize.html index 3813f981..cacbf6e5 100644 --- a/doc/rsconf1_dynafilecachesize.html +++ b/doc/rsconf1_dynafilecachesize.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$DynaFileCacheSize</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> 10</p> @@ -20,4 +22,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_escapecontrolcharactersonreceive.html b/doc/rsconf1_escapecontrolcharactersonreceive.html index 26917736..178f9a6f 100644 --- a/doc/rsconf1_escapecontrolcharactersonreceive.html +++ b/doc/rsconf1_escapecontrolcharactersonreceive.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$EscapeControlCharactersOnReceive</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> on</p> diff --git a/doc/rsconf1_failonchownfailure.html b/doc/rsconf1_failonchownfailure.html index 0e646e36..d8bbab82 100644 --- a/doc/rsconf1_failonchownfailure.html +++ b/doc/rsconf1_failonchownfailure.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$FailOnChownFailure</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> on</p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_filecreatemode.html b/doc/rsconf1_filecreatemode.html index c8440864..10b0317b 100644 --- a/doc/rsconf1_filecreatemode.html +++ b/doc/rsconf1_filecreatemode.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$FileCreateMode</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> 0644</p> diff --git a/doc/rsconf1_filegroup.html b/doc/rsconf1_filegroup.html index b9acaab7..dd5b8ad5 100644 --- a/doc/rsconf1_filegroup.html +++ b/doc/rsconf1_filegroup.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$FileGroup</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_fileowner.html b/doc/rsconf1_fileowner.html index 7a9cbbc7..935cfffd 100644 --- a/doc/rsconf1_fileowner.html +++ b/doc/rsconf1_fileowner.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$FileOwner</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_generateconfiggraph.html b/doc/rsconf1_generateconfiggraph.html new file mode 100644 index 00000000..0b18463a --- /dev/null +++ b/doc/rsconf1_generateconfiggraph.html @@ -0,0 +1,121 @@ +<html> +<head> +<title>rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">back</a> + +<h2>$GenerateConfigGraph</h2> +<p><b>Type:</b> global configuration directive</p> +<p><b>Default:</b> </p> +<p><b>Available Since:</b> 4.3.1</p> +<p><b>Description:</b></p> +<p>This directive permits to create (hopefully) good-looking visualizations of rsyslogd's +configuration. It does not affect rsyslog operation. If the directive is specified multiple +times, all but the last are ignored. If it is specified, a graph is created. This happens +both during a regular startup as well a config check run. It is recommended to include +this directive only for documentation purposes and remove it from a production +configuraton. +<p>The graph is not drawn by rsyslog itself. Instead, it uses the great open source tool +<a href="http://www.graphviz.org">Graphviz</a> to do the actual drawing. This has at least +two advantages: +<ul> +<li>the graph drawing support code in rsyslog is extremly slim and without overhead +<li>the user may change or further annotate the generated file, thus potentially +improving his documentation +</ul> +The drawback, of course, is that you need to run Graphviz once you have generated +the control file with rsyslog. Fortunately, the process to do so is rather easy: +<ol> +<li>add "$GenerateConfigGraph /path/to/file.dot" to rsyslog.conf (from now on, I +will call the file just file.dot). Optionally, add "$ActionName" statement +<b>in front of</b> those actions that you like to use friendly names with. If you do +this, keep the names short. +<li>run rsyslog at least once (either in regular or configuration check mode) +<li>remember to remove the $GenerateConfigGraph directive when you no longer need it (or +comment it out) +<li>change your working directory to where you place the dot file +<li>if you would like to edit the rsyslog-generated file, now is the time to do so +<li>do "dot -Tpng file.dot > file.png" +<li>remember that you can use "convert -resize 50% file.png resized.png" if +dot's output is too large (likely) or too small. Resizing can be especially useful if +you intend to get a rough overview over your configuration. +</ol> +After completing these steps, you should have a nice graph of your configuration. Details +are missing, but that is exactly the point. At the start of the graph is always (at least +in this version, could be improved) a node called "inputs" in a tripple hexagon +shape. This represents all inputs active in the system (assuming you have defined some, +what the current version does not check). Next comes the main queue. It is given in a +hexagon shape. That shape indicates that a queue is peresent and used to de-couple +the inbound from the outbound part of the graph. In technical terms, here is a +threading boundary. Action with "real" queues (other than in direct mode) +also utilize this shape. For actions, notice that a "hexagon action" creates +a deep copy of the message. As such, a "discard hexagon action" actually does +nothing, because it duplicates the message and then discards <b>the duplicate</b>. +At the end of the diagram, you always see a "discard" action. This indicates +that rsyslog discards messages which have been run through all available rules. +<p>Edges are labeled with information about when they are taken. For filters, the type of +filter, but not any specifics, are given. It is also indicated if no filter is +applied in the configuration file (by using a "*.*" selector). Edges without +labels are unconditionally taken. The actions themselfs are labeled with the name of +the output module that handles them. If provided, the name given via +"ActionName" is used instead. No further details are provided. +<p>If there is anything in red, this should draw your attention. In this case, rsyslogd +has detected something that does not look quite right. A typical example is a discard +action which is followed by some other actions in an action unit. Even though something +may be red, it can be valid - rsyslogd's graph generator does not yet check each and +every speciality, so the configuration may just cover a very uncommon case. +<p>Now let's look at some examples. The graph below was generated on a fairly standard +Fedora rsyslog.conf file. It had only the usually commented-out last forwarding action +activated: +<p align="center"> +<img src="rsyslog_confgraph_std.png" alt="rsyslog configuration graph for a default fedora rsyslog.conf"> +<p>This is the typical structure for a simple rsyslog configuration. There are a couple of +actions, each guarded by a filter. Messages run from top to bottom and control branches +whenever a filter evaluates to true. As there is no discard action, all messages will +run through all filters and discarded in the system default discard action right after +all configured actions. +</p> +<p>A more complex example can be seen in the next graph. This is a configuration I +created for testing the graph-creation features, so it contains a little bit of +everything. However, real-world configurations can look quite complex, too (and I +wouldn't say this one is very complex): +<p align="center"> +<img src="rsyslog_confgraph_complex.png"> +</p> +<p>Here, we have a user-defined discard action. You can immediately see this because +processing branches after the first "builtin-file" action. Those messages +where the filter evaluates to true for will never run through the left-hand action +branch. However, there is also a configuration error present: there are two more +actions (now shown red) after the discard action. As the message is discarded, these will +never be executed. Note that the discard branch contains no further filters. This is +because these actions are all part of the same action unit, which is guarded only by +an entry filter. The same is present a bit further down at the node labeled +"write_system_log_2". This note has one more special feature, that is label +was set via "ActionName", thus is does not have standard form (the same +happened to the node named "Forward" right at the top of the diagram. +Inside this diagram, the "Forward" node is executed asynchonously on its own +queue. All others are executed synchronously. +<p>Configuration graphs are useful for documenting a setup, but are also a great +<a href="troubleshoot.html">troubleshooting</a> resource. It is important to +remember that <b>these graphs are generated +from rsyslogd's in-memory action processing structures</b>. You can not get closer +to understanding on how rsyslog interpreted its configuration files. +So if the graph does not look +what you intended to do, there is probably something worng in rsyslog.conf. +<p>If something is not working as expected, but you do not spot the error immediately, +I recommend to generate a graph and zoom it so that you see all of it in one great picture. +You may not be able to read anything, but the structure should look good to you and +so you can zoom into those areas that draw your attention. +<p><b>Sample:</b></p> +<p><code><b>$DirOwner /path/to/graphfile-file.dot</b></code></p> + +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_gssforwardservicename.html b/doc/rsconf1_gssforwardservicename.html index 9d39dc2a..45d9ba98 100644 --- a/doc/rsconf1_gssforwardservicename.html +++ b/doc/rsconf1_gssforwardservicename.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$GssForwardServiceName</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> host</p> diff --git a/doc/rsconf1_gsslistenservicename.html b/doc/rsconf1_gsslistenservicename.html index cd03dc58..5fdf3edc 100644 --- a/doc/rsconf1_gsslistenservicename.html +++ b/doc/rsconf1_gsslistenservicename.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$GssListenServiceName</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> host</p> diff --git a/doc/rsconf1_gssmode.html b/doc/rsconf1_gssmode.html index 71c50696..2b1d5656 100644 --- a/doc/rsconf1_gssmode.html +++ b/doc/rsconf1_gssmode.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$GssMode</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> encryption</p> diff --git a/doc/rsconf1_includeconfig.html b/doc/rsconf1_includeconfig.html index 24462f77..132cee6f 100644 --- a/doc/rsconf1_includeconfig.html +++ b/doc/rsconf1_includeconfig.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$IncludeConfig</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> @@ -43,4 +45,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_mainmsgqueuesize.html b/doc/rsconf1_mainmsgqueuesize.html index acf88e94..ffed1c09 100644 --- a/doc/rsconf1_mainmsgqueuesize.html +++ b/doc/rsconf1_mainmsgqueuesize.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$MainMsgQueueSize</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> 10000</p> diff --git a/doc/rsconf1_markmessageperiod.html b/doc/rsconf1_markmessageperiod.html index 9b6590cd..2c833339 100644 --- a/doc/rsconf1_markmessageperiod.html +++ b/doc/rsconf1_markmessageperiod.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$MarkMessagePeriod</h2> <p><b>Type:</b> specific to immark input module</p> <p><b>Default:</b> 1800 (20 minutes)</p> @@ -27,4 +29,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_maxopenfiles.html b/doc/rsconf1_maxopenfiles.html new file mode 100644 index 00000000..b6c9cc0e --- /dev/null +++ b/doc/rsconf1_maxopenfiles.html @@ -0,0 +1,35 @@ +<html> +<head> +<title>$MaxOpenFiles - rsyslog.conf file</title> +</head> +<body> +<a href="rsyslog_conf_global.html">[rsyslog configuration directive overview]</a> + +<h2>$MaxOpenFiles</h2> +<p><b>Available Since:</b> 4.3.0</p> +<p><b>Type:</b> global configuration directive</p> +<p><b>Default:</b> <i>operating system default</i></p> +<p><b>Description:</b></p> +<p>Set the maximum number of files that the rsyslog process can have open at any given +time. Note that this includes open tcp sockets, so this setting is the upper limit for +the number of open TCP connections as well. If you expect a large nubmer of concurrent +connections, it is suggested that the number is set to the max number connected plus 1000. +Please note that each dynafile also requires up to 100 open file handles. +<p>The setting is similar to running "ulimit -n number-of-files". +<p>Please note that depending on permissions and operating system configuration, the +setrlimit() request issued by rsyslog may fail, in which case the previous limit is kept +in effect. Rsyslog will emit a warning message in this case. +<p><b>Sample:</b></p> +<p><code><b>$MaxOpenFiles 2000</b></code></p> +<p><b>Bugs:</b></p> +<p>For some reason, this settings seems not to work on all platforms. If you experience +problems, please let us know so that we can (hopefully) narrow down the issue. +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] [<a href="manual.html">manual +index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> diff --git a/doc/rsconf1_moddir.html b/doc/rsconf1_moddir.html index ced07dc9..889de05d 100644 --- a/doc/rsconf1_moddir.html +++ b/doc/rsconf1_moddir.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$ModDir</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> system default for user libraries, e.g. @@ -24,4 +26,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_modload.html b/doc/rsconf1_modload.html index a2b8087a..ce457ea5 100644 --- a/doc/rsconf1_modload.html +++ b/doc/rsconf1_modload.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$ModLoad</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> diff --git a/doc/rsconf1_repeatedmsgreduction.html b/doc/rsconf1_repeatedmsgreduction.html index 20e56f89..248e8343 100644 --- a/doc/rsconf1_repeatedmsgreduction.html +++ b/doc/rsconf1_repeatedmsgreduction.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$RepeatedMsgReduction</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> depending on -e</p> @@ -20,4 +22,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_resetconfigvariables.html b/doc/rsconf1_resetconfigvariables.html index 9794d158..46cf0bdf 100644 --- a/doc/rsconf1_resetconfigvariables.html +++ b/doc/rsconf1_resetconfigvariables.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$ResetConfigVariables</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> @@ -19,4 +21,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rsconf1_umask.html b/doc/rsconf1_umask.html index ee47dbad..8e41e672 100644 --- a/doc/rsconf1_umask.html +++ b/doc/rsconf1_umask.html @@ -3,6 +3,8 @@ <title>rsyslog.conf file</title> </head> <body> +<a href="rsyslog_conf_global.html">back</a> + <h2>$UMASK</h2> <p><b>Type:</b> global configuration directive</p> <p><b>Default:</b> </p> @@ -21,4 +23,4 @@ Copyright © 2007 by <a href="http://www.gerhards.net/rainer">Rainer Gerhard <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> -</html>
\ No newline at end of file +</html> diff --git a/doc/rscript_abnf.html b/doc/rscript_abnf.html index 278fb59c..d60edb5c 100644 --- a/doc/rscript_abnf.html +++ b/doc/rscript_abnf.html @@ -30,6 +30,8 @@ table... values('&$facility&','&$severity&...?]<br> endact table... values('&$facility&','&$severity&...?]<br> )<br><br>... or ...</p><p>define action writeMySQL(<br> type='ommysql.so', queue.mode='disk', queue.highwatermark = 300,<br> action.dbname='events', action.dbuser='uid',<br> [?action.template='templatename'?] or [?action.sql='insert into table... values('&$facility&','&$severity&...?]<br> )</p><p>if $message contains "error" then action writeMySQL(action.dbname='differentDB')</p><p></p><p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +<h2>Implementation</h2> +RainerScript will be implemented via a hand-crafted LL(1) parser. I was tempted to use yacc, but it turned out the resulting code was not thread-safe and as such did not fit within the context of rsyslog. Also, limited error handling is not a real problem for us: if there is a problem in parsing the configuration file, we stop processing. Guessing what was meant and trying to recover would IMHO not be good choices for something like a syslogd. [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> diff --git a/doc/rsyslog-vers.dot b/doc/rsyslog-vers.dot new file mode 100644 index 00000000..a5563f94 --- /dev/null +++ b/doc/rsyslog-vers.dot @@ -0,0 +1,82 @@ +// This file is part of rsyslog. +// +// rsyslog "family tree" compressed version +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot rsyslog-vers.dot -Tpng >rsyslog-vers.png + +digraph G { + label="\n\nrsyslog \"family tree\"\nhttp://www.rsyslog.com"; + fontsize=20; + + v1stable [label="v1-stable", shape=box, style=dotted]; + v2stable [label="v2-stable", shape=box, style=filled]; + v3stable [label="v3-stable", shape=box, style=filled]; + beta [label="beta", shape=box, style=filled]; + devel [label="devel", shape=box, style=filled]; + "1.0.5" [style=dotted]; + perf [style=dashed]; + "imudp-select" [style=dashed label="imudp-\nselect"]; + msgnolock [style=dashed]; + "file-errHdlr" [style=dashed]; + solaris [style=dashed]; + tests [style=dashed]; + "0.x.y" [group=master]; + "1.10.0" [group=master]; + "1.21.2" [group=master]; + "3.10.0" [group=master]; + "3.15.1" [group=master]; + "3.17.0" [group=master]; + "3.19.x" [group=master]; + "3.21.x" [group=master]; + "4.1.0" [group=master]; + "4.1.4" [group=master]; + "4.1.5" [group=master]; + "4.1.6" [group=master]; + + sysklogd -> "0.x.y" [color=red]; + "0.x.y" -> "1.0.0"; + "0.x.y" -> "1.10.0" [color=red]; + "1.0.0" -> "1.0.5" [style=dashed]; + "1.10.0" -> "1.21.2" [color=red style=dashed]; + "1.21.2" -> "2.0.0" [color=blue]; + "2.0.0" -> "2.0.5" [style=dashed, color=blue]; + "1.21.2" -> "3.10.0" [color=red]; + "3.10.0" -> "3.15.1" [color=red style=dashed]; + "3.15.1" -> "tests"; + "3.15.1" -> "3.17.0" [color=red style=dashed]; + "3.15.1" -> "3.16.x"; + "3.16.x" -> "3.18.x" [color=blue, style=dashed]; + "3.17.0" -> "3.18.x"; + "3.17.0" -> "3.19.x" [color=red, style=dashed]; + "3.19.x" -> "3.20.x"; + "3.19.x" -> "3.21.x" [color=red]; + "3.18.x" -> debian_lenny; + "3.18.x" -> "3.20.x" [color=blue, style=dashed]; + "3.21.x" -> "3.21.11"; + "3.21.x" -> "4.1.0" [color=red]; + "3.21.x" -> "perf"; + "perf" -> "4.1.0"; + "4.1.0" -> "4.1.4" [color=red, style=dashed]; + "4.1.4" -> "file-errHdlr"; + "4.1.4" -> "4.1.5" [color=red]; + "4.1.5" -> "4.1.6" [color=red]; + "3.21.x" -> msgnolock + "3.21.x" -> "imudp-select"; + "4.1.5" -> solaris; + "file-errHdlr" -> "4.1.6"; + "tests" -> "4.1.6"; + + "1.0.5" -> v1stable [dir=none, style=dotted]; + "2.0.5" -> v2stable [dir=none, style=dotted]; + "3.20.x" -> v3stable [dir=none, style=dotted]; + "3.21.11" -> beta [dir=none, style=dotted]; + "4.1.6" -> devel [dir=none, style=dotted]; + + {rank=same; "4.1.5" "solaris"} + {rank=same; "3.18.x" "debian_lenny"} + {rank=same; "1.0.5" "2.0.5" "3.20.x" "3.21.11" "4.1.6"} +} diff --git a/doc/rsyslog-vers.png b/doc/rsyslog-vers.png Binary files differnew file mode 100644 index 00000000..e8ec8b81 --- /dev/null +++ b/doc/rsyslog-vers.png diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 1a24e793..6990c6bd 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -20,241 +20,13 @@ possible. While, for obvious reasons, <a href="features.html">enhanced features</a> require a different config file syntax, rsyslogd should be able to work with a standard syslog.conf file. This is especially useful while you are migrating from syslogd to rsyslogd.</p> -<h2>Modules</h2> -<p>Rsyslog has a modular design. Consequently, there is a growing -number of modules. Here is the entry point to their documentation and -what they do (list is currently not complete)</p> -<ul> -<li><a href="omsnmp.html">omsnmp</a> - SNMP -trap output module</li> -<li><a href="omrelp.html">omrelp</a> - RELP -output module</li> -<li>omgssapi - output module for GSS-enabled syslog</li> -<li><a href="ommysql.html">ommysql</a> - output module for MySQL</li> -<li>ompgsql - output module for PostgreSQL</li> -<li><a href="omlibdbi.html">omlibdbi</a> - -generic database output module (Firebird/Interbase, MS SQL, Sybase, -SQLLite, Ingres, Oracle, mSQL)</li> -<li><a href="ommail.html">ommail</a> - -permits rsyslog to alert folks by mail if something important happens</li> -<li><a href="imfile.html">imfile</a> -- input module for text files</li> -<li><a href="imrelp.html">imrelp</a> - RELP -input module</li> -<li>imudp - udp syslog message input</li> -<li><a href="imtcp.html">imtcp</a> - input -plugin for plain tcp syslog</li> -<li><a href="imgssapi.html">imgssapi</a> - -input plugin for plain tcp and GSS-enabled syslog</li> -<li>immark - support for mark messages</li> -<li><a href="imklog.html">imklog</a> - kernel logging</li> -<li><a href="imuxsock.html">imuxsock</a> - -unix sockets, including the system log socket</li> -<li><a href="im3195.html">im3195</a> - -accepts syslog messages via RFC 3195</li> -</ul> -<p>Please note that each module provides configuration -directives, which are NOT necessarily being listed below. Also -remember, that a modules configuration directive (and functionality) is -only available if it has been loaded (using $ModLoad).</p> +<h2><a href="rsyslog_conf_modules.html">Modules</a></h2> <h2>Lines</h2> Lines can be continued by specifying a backslash ("\") as the last -character of the line.<br> -<h2>Global Directives</h2> -<p>All global directives need to be specified on a line by their -own and must start with a dollar-sign. Here is a list in alphabetical -order. Follow links for a description.</p> -<p>Not all directives have an in-depth description right now. -Default values for them are in bold. A more in-depth description will -appear as implementation progresses. Directives may change during that -process, we will NOT try hard to maintain backwards compatibility -(after all, v3 is still very early in development and quite -unstable...). So you have been warned ;)</p> -<p><b>Be sure to read information about <a href="queues.html">queues in rsyslog</a></b> - -many parameter settings modify queue parameters. If in doubt, use the -default, it is usually well-chosen and applicable in most cases.</p> -<ul> -<li><a href="rsconf1_actionexeconlywhenpreviousissuspended.html">$ActionExecOnlyWhenPreviousIsSuspended</a></li> -<li>$ActionExecOnlyOnceEveryInterval <seconds> - -execute action only if the last execute is at last -<seconds> seconds in the past (more info in <a href="ommail.html">ommail</a>, -but may be used with any action)</li> -<li><i><b>$ActionExecOnlyEveryNthTime</b> <number></i> - If configured, the next action will -only be executed every n-th time. For example, if configured to 3, the first two messages -that go into the action will be dropped, the 3rd will actually cause the action to execute, -the 4th and 5th will be dropped, the 6th executed under the action, ... and so on. Note: -this setting is automatically re-set when the actual action is defined.</li> -<li><i><b>$ActionExecOnlyEveryNthTimeTimeout</b> <number-of-seconds></i> - has a meaning only if -$ActionExecOnlyEveryNthTime is also configured for the same action. If so, the timeout -setting specifies after which period the counting of "previous actions" expires and -a new action count is begun. Specify 0 (the default) to disable timeouts. -<br> -<i>Why is this option needed?</i> Consider this case: a message comes in at, eg., 10am. That's -count 1. Then, nothing happens for the next 10 hours. At 8pm, the next -one occurs. That's count 2. Another 5 hours later, the next message -occurs, bringing the total count to 3. Thus, this message now triggers -the rule. -<br> -The question is if this is desired behavior? Or should the rule only be -triggered if the messages occur within an e.g. 20 minute window? If the -later is the case, you need a -<br> -$ActionExecOnlyEveryNthTimeTimeout 1200 -<br> -This directive will timeout previous messages seen if they are older -than 20 minutes. In the example above, the count would now be always 1 -and consequently no rule would ever be triggered. - -<li>$ActionFileDefaultTemplate [templateName] - sets a new default template for file actions</li> -<li>$ActionFileEnableSync [on/<span style="font-weight: bold;">off</span>] - enables file -syncing capability of omfile</li> -<li>$ActionForwardDefaultTemplate [templateName] - sets a new -default template for UDP and plain TCP forwarding action</li> -<li>$ActionGSSForwardDefaultTemplate [templateName] - sets a -new default template for GSS-API forwarding action</li> -<li>$ActionQueueCheckpointInterval <number></li> -<li>$ActionQueueDequeueSlowdown <number> [number -is timeout in <i> micro</i>seconds (1000000us is 1sec!), -default 0 (no delay). Simple rate-limiting!]</li> -<li>$ActionQueueDiscardMark <number> [default -9750]</li> -<li>$ActionQueueDiscardSeverity <number> -[*numerical* severity! default 4 (warning)]</li> -<li>$ActionQueueFileName <name></li> -<li>$ActionQueueHighWaterMark <number> [default -8000]</li> -<li>$ActionQueueImmediateShutdown [on/<b>off</b>]</li> -<li>$ActionQueueSize <number></li> -<li>$ActionQueueLowWaterMark <number> [default -2000]</li> -<li>$ActionQueueMaxFileSize <size_nbr>, default 1m</li> -<li>$ActionQueueTimeoutActionCompletion <number> -[number is timeout in ms (1000ms is 1sec!), default 1000, 0 means -immediate!]</li> -<li>$ActionQueueTimeoutEnqueue <number> [number -is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li> -<li>$ActionQueueTimeoutShutdown <number> [number -is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li> -<li>$ActionQueueWorkerTimeoutThreadShutdown -<number> [number is timeout in ms (1000ms is 1sec!), -default 60000 (1 minute)]</li> -<li>$ActionQueueType [FixedArray/LinkedList/<b>Direct</b>/Disk]</li> -<li>$ActionQueueSaveOnShutdown [on/<b>off</b>] -</li> -<li>$ActionQueueWorkerThreads <number>, num worker threads, default 1, recommended 1</li> -<li>$ActionQueueWorkerThreadMinumumMessages <number>, default 100</li> -<li><a href="rsconf1_actionresumeinterval.html">$ActionResumeInterval</a></li> -<li>$ActionResumeRetryCount <number> [default 0, -1 means eternal]</li> -<li>$ActionSendResendLastMsgOnReconn <[on/<b>off</b>]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. -<li>$ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action -</li><li>$ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver -(driver-specific)</li><li>$ActionSendStreamDriverAuthMode <mode>, authentication mode to use with the stream driver -(driver-specific)</li><li>$ActionSendStreamDriverPermittedPeer <ID>, accepted fingerprint (SHA1) or name of remote peer -(driver-specific) -<span style="font-weight: bold;"> directive may go away</span>!</li> -<li><a href="rsconf1_allowedsender.html">$AllowedSender</a></li> -<li><a href="rsconf1_controlcharacterescapeprefix.html">$ControlCharacterEscapePrefix</a></li> -<li><a href="rsconf1_debugprintcfsyslinehandlerlist.html">$DebugPrintCFSyslineHandlerList</a></li> - -<li><a href="rsconf1_debugprintmodulelist.html">$DebugPrintModuleList</a></li> -<li><a href="rsconf1_debugprinttemplatelist.html">$DebugPrintTemplateList</a></li> -<li>$DefaultNetstreamDriver <drivername>, the default <a href="netstream.html">network stream driver</a> to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem></li> -<li>$DefaultNetstreamDriverCertFile </path/to/certfile.pem></li> -<li>$DefaultNetstreamDriverKeyFile </path/to/keyfile.pem></li> -<li><a href="rsconf1_dircreatemode.html">$DirCreateMode</a></li> -<li><a href="rsconf1_dirgroup.html">$DirGroup</a></li> -<li><a href="rsconf1_dirowner.html">$DirOwner</a></li> -<li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li> -<li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li> -<li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li> -<li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li> -<li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> -<li><a href="rsconf1_failonchownfailure.html">$FailOnChownFailure</a></li> -<li><a href="rsconf1_filecreatemode.html">$FileCreateMode</a></li> -<li><a href="rsconf1_filegroup.html">$FileGroup</a></li> -<li><a href="rsconf1_fileowner.html">$FileOwner</a></li> -<li><a href="rsconf1_gssforwardservicename.html">$GssForwardServiceName</a></li> -<li><a href="rsconf1_gsslistenservicename.html">$GssListenServiceName</a></li> -<li><a href="rsconf1_gssmode.html">$GssMode</a></li> -<li><a href="rsconf1_includeconfig.html">$IncludeConfig</a></li><li>MainMsgQueueCheckpointInterval <number></li> -<li>$MainMsgQueueDequeueSlowdown <number> [number -is timeout in <i> micro</i>seconds (1000000us is 1sec!), -default 0 (no delay). Simple rate-limiting!]</li> -<li>$MainMsgQueueDiscardMark <number> [default -9750]</li> -<li>$MainMsgQueueDiscardSeverity <severity> -[either a textual or numerical severity! default 4 (warning)]</li> -<li>$MainMsgQueueFileName <name></li> -<li>$MainMsgQueueHighWaterMark <number> [default -8000]</li> -<li>$MainMsgQueueImmediateShutdown [on/<b>off</b>]</li> -<li><a href="rsconf1_mainmsgqueuesize.html">$MainMsgQueueSize</a></li> -<li>$MainMsgQueueLowWaterMark <number> [default -2000]</li> -<li>$MainMsgQueueMaxFileSize <size_nbr>, default -1m</li> -<li>$MainMsgQueueTimeoutActionCompletion -<number> [number is timeout in ms (1000ms is 1sec!), -default -1000, 0 means immediate!]</li> -<li>$MainMsgQueueTimeoutEnqueue <number> [number -is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li> -<li>$MainMsgQueueTimeoutShutdown <number> [number -is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li> -<li>$MainMsgQueueWorkerTimeoutThreadShutdown -<number> [number is timeout in ms (1000ms is 1sec!), -default 60000 (1 minute)]</li> -<li>$MainMsgQueueType [<b>FixedArray</b>/LinkedList/Direct/Disk]</li> -<li>$MainMsgQueueSaveOnShutdown [on/<b>off</b>] -</li> -<li>$MainMsgQueueWorkerThreads <number>, num -worker threads, default 1, recommended 1</li> -<li>$MainMsgQueueWorkerThreadMinumumMessages <number>, default 100</li> -<li><a href="rsconf1_markmessageperiod.html">$MarkMessagePeriod</a> (immark)</li> -<li><b><i>$MaxMessageSize</i></b> <size_nbr>, default 2k - allows to specify maximum supported message size -(both for sending and receiving). The default -should be sufficient for almost all cases. Do not set this below 1k, as it would cause -interoperability problems with other syslog implementations.<br> -Change the setting to e.g. 32768 if you would like to -support large message sizes for IHE (32k is the current maximum -needed for IHE). I was initially tempted to set the default to 32k, -but there is a some memory footprint with the current -implementation in rsyslog. -<br>If you intend to receive Windows Event Log data (e.g. via -<a href="http://www.eventreporter.com/">EventReporter</a>), you might want to -increase this number to an even higher value, as event -log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea). -Note: testing showed that 4k seems to be -the typical maximum for <b>UDP</b> based syslog. This is an IP stack -restriction. Not always ... but very often. If you go beyond -that value, be sure to test that rsyslogd actually does what -you think it should do ;) It is highly suggested to use a TCP based transport -instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions. -<br>Note that 2k, the current default, is the smallest size that must be -supported in order to be compliant to the upcoming new syslog RFC series. -</li> -<li><a href="rsconf1_moddir.html">$ModDir</a></li> -<li><a href="rsconf1_modload.html">$ModLoad</a></li> -<li><a href="rsconf1_repeatedmsgreduction.html">$RepeatedMsgReduction</a></li> -<li><a href="rsconf1_resetconfigvariables.html">$ResetConfigVariables</a></li> -<li>$WorkDirectory <name> (directory for spool -and other work files)</li> -<li>$UDPServerAddress <IP> (imudp) -- local IP -address (or name) the UDP listens should bind to</li> -<li>$UDPServerRun <port> (imudp) -- former --r<port> option, default 514, start UDP server on this -port, "*" means all addresses</li> -<li><a href="rsconf1_umask.html">$UMASK</a></li> -</ul> -<p><b>Where <size_nbr> is specified above,</b> -modifiers can be used after the number part. For example, 1k means -1024. Supported are k(ilo), m(ega), g(iga), t(era), p(eta) and e(xa). -Lower case letters refer to the traditional binary defintion (e.g. 1m -equals 1,048,576) whereas upper case letters refer to their new -1000-based definition (e.g 1M equals 1,000,000).</p> -<p>Numbers may include '.' and ',' for readability. So you can -for example specify either "1000" or "1,000" with the same result. -Please note that rsyslogd simply ignores the punctuation. Form it's -point of view, "1,,0.0.,.,0" also has the value 1000. </p> +character of the line. There is a hard-coded maximum line length of 4K. +If you need lines larger than that, you need to change compile-time +settings inside rsyslog and recompile. +<h2><a href="rsyslog_conf_global.html">Configuration Directives</a></h2> <h2>Basic Structure</h2> <p>Rsyslog supports standard sysklogd's configuration file format and extends it. So in general, you can take a "normal" syslog.conf and @@ -273,977 +45,15 @@ priorities belonging to the specified action.<br> <br> Lines starting with a hash mark ("#'') and empty lines are ignored. </p> -<h2>Templates</h2> -<p>Templates are a key feature of rsyslog. They allow to specify -any -format a user might want. They are also used for dynamic file name -generation. Every output in rsyslog uses templates - this holds true -for files, user messages and so on. -Please note that there is an -<a href="http://www.rsyslog.com/Article354.phtml">online tutorial on rsyslog templates</a> -available on the web. We recommend viewing it. -The database writer expects its -template to be a proper SQL statement - so this is highly customizable -too. You might ask how does all of this work when no templates at all -are specified. Good question ;) The answer is simple, though. Templates -compatible with the stock syslogd formats are hardcoded into rsyslogd. -So if no template is specified, we use one of these hardcoded -templates. Search for "template_" in syslogd.c and you will find the -hardcoded ones.</p> -<p>A template consists of a template directive, a name, the -actual template text and optional options. A sample is:</p> -<blockquote><code>$template MyTemplateName,"\7Text -%property% some more text\n",<options></code></blockquote> -<p>The "$template" is the template directive. It tells rsyslog -that this line contains a template. "MyTemplateName" is the template -name. All -other config lines refer to this name. The text within quotes is the -actual template text. The backslash is an escape character, much as it -is in C. It does all these "cool" things. For example, \7 rings the -bell (this is an ASCII value), \n is a new line. C programmers and perl -coders have the advantage of knowing this, but the set in rsyslog is a -bit restricted currently. -</p> -<p>All text in the template is used literally, except for things -within percent signs. These are properties and allow you access to the -contents of the syslog message. Properties are accessed via the -property replacer (nice name, huh) and it can do cool things, too. For -example, it can pick a substring or do date-specific formatting. More -on this is below, on some lines of the property replacer.<br> -<br> -The <options> part is optional. It carries options -influencing the template as whole. See details below. Be sure NOT to -mistake template options with property options - the later ones are -processed by the property replacer and apply to a SINGLE property, only -(and not the whole template).<br> -<br> -Template options are case-insensitive. Currently defined are: </p> -<p><b>sql</b> - format the string suitable for a SQL -statement in MySQL format. This will replace single quotes ("'") and -the backslash character by their backslash-escaped counterpart ("\'" -and "\\") inside each field. Please note that in MySQL configuration, -the <code class="literal">NO_BACKSLASH_ESCAPES</code> -mode must be turned off for this format to work (this is the default).</p> -<p><b>stdsql</b> - format the string suitable for a -SQL statement that is to be sent to a standards-compliant sql server. -This will replace single quotes ("'") by two single quotes ("''") -inside each field. You must use stdsql together with MySQL if in MySQL -configuration the -<code class="literal">NO_BACKSLASH_ESCAPES</code> is -turned on.</p> -<p>Either the <b>sql</b> or <b>stdsql</b> -option <b>must</b> be specified when a template is used -for writing to a database, otherwise injection might occur. Please note -that due to the unfortunate fact that several vendors have violated the -sql standard and introduced their own escape methods, it is impossible -to have a single option doing all the work. So you yourself -must make sure you are using the right format. <b>If you choose -the wrong one, you are still vulnerable to sql injection.</b><br> -<br> -Please note that the database writer *checks* that the sql option is -present in the template. If it is not present, the write database -action is disabled. This is to guard you against accidental forgetting -it and then becoming vulnerable to SQL injection. The sql option can -also be useful with files - especially if you want to import them into -a database on another machine for performance reasons. However, do NOT -use it if you do not have a real need for it - among others, it takes -some toll on the processing time. Not much, but on a really busy system -you might notice it ;)</p> -<p>The default template for the write to database action has the -sql option set. As we currently support only MySQL and the sql option -matches the default MySQL configuration, this is a good choice. -However, if you have turned on -<code class="literal">NO_BACKSLASH_ESCAPES</code> in -your MySQL config, you need to supply a template with the stdsql -option. Otherwise you will become vulnerable to SQL injection. <br> -<br> -To escape:<br> -% = \%<br> -\ = \\ --> '\' is used to escape (as in C)<br> -$template TraditionalFormat,%timegenerated% %HOSTNAME% -%syslogtag%%msg%\n"<br> -<br> -Properties can be accessed by the <a href="property_replacer.html">property -replacer</a> (see there for details).</p> -<p><b>Please note that templates can also by -used to generate selector lines with dynamic file names.</b> For -example, if you would like to split syslog messages from different -hosts to different files (one per host), you can define the following -template:</p> -<blockquote><code>$template -DynFile,"/var/log/system-%HOSTNAME%.log"</code></blockquote> -<p>This template can then be used when defining an output -selector line. It will result in something like -"/var/log/system-localhost.log"</p> -<p>Template -names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT -use them if, otherwise you may receive a conflict in the future (and -quite unpredictable behaviour). There is a small set of pre-defined -templates that you can use without the need to define it:</p> -<ul> -<li><span style="font-weight: bold;">RSYSLOG_TraditionalFileFormat</span> -- the "old style" default log file format with low-precision timestamps</li> -<li><span style="font-weight: bold;">RSYSLOG_FileFormat</span> -- a modern-style logfile format similar to TraditionalFileFormat, buth -with high-precision timestamps and timezone information</li> -<li><span style="font-weight: bold;">RSYSLOG_TraditionalForwardFormat</span> -- the traditional forwarding format with low-precision timestamps. Most -useful if you send messages to other syslogd's or rsyslogd -below -version 3.12.5.</li> -<li><span style="font-weight: bold;">RSYSLOG_ForwardFormat</span> -- a new high-precision forwarding format very similar to the -traditional one, but with high-precision timestamps and timezone -information. Recommended to be used when sending messages to rsyslog -3.12.5 or above.</li> -<li><span style="font-weight: bold;">RSYSLOG_SyslogProtocol23Format</span> -- the format specified in IETF's internet-draft -ietf-syslog-protocol-23, which is assumed to be come the new syslog -standard RFC. This format includes several improvements. The rsyslog -message parser understands this format, so you can use it together with -all relatively recent versions of rsyslog. Other syslogd's may get -hopelessly confused if receiving that format, so check before you use -it. Note that the format is unlikely to change when the final RFC comes -out, but this may happen.</li> -<li><span style="font-weight: bold;">RSYSLOG_DebugFormat</span> -- a special format used for troubleshooting property problems. This format -is meant to be written to a log file. Do <b>not</b> use for production or remote -forwarding.</li> -</ul> -<h2>Output Channels</h2> -<p>Output Channels are a new concept first introduced in rsyslog -0.9.0. <b>As of this writing, it is most likely that they will -be replaced by something different in the future.</b> So if you -use them, be prepared to change you configuration file syntax when you -upgrade to a later release.<br> -<br> -The idea behind output channel definitions is that it shall provide an -umbrella for any type of output that the user might want. In essence,<br> -this is the "file" part of selector lines (and this is why we are not -sure output channel syntax will stay after the next review). There is a<br> -difference, though: selector channels both have filter conditions -(currently facility and severity) as well as the output destination. -Output channels define the output definition, only. As of this build, -they can only be used to write to files - not pipes, ttys or whatever -else. If we stick with output channels, this will change over time.</p> -<p>In concept, an output channel includes everything needed to -know about an output actions. In practice, the current implementation -only carries<br> -a filename, a maximum file size and a command to be issued when this -file size is reached. More things might be present in future version, -which might also change the syntax of the directive.</p> -<p>Output channels are defined via an $outchannel directive. It's -syntax is as follows:<br> -<br> -$outchannel name,file-name,max-size,action-on-max-size<br> -<br> -name is the name of the output channel (not the file), file-name is the -file name to be written to, max-size the maximum allowed size and -action-on-max-size a command to be issued when the max size is reached. -This command always has exactly one parameter. The binary is that part -of action-on-max-size before the first space, its parameter is -everything behind that space.<br> -<br> -Please note that max-size is queried BEFORE writing the log message to -the file. So be sure to set this limit reasonably low so that any -message might fit. For the current release, setting it 1k lower than -you expected is helpful. The max-size must always be specified in bytes -- there are no special symbols (like 1k, 1m,...) at this point of -development.<br> -<br> -Keep in mind that $outchannel just defines a channel with "name". It -does not activate it. To do so, you must use a selector line (see -below). That selector line includes the channel name plus an $ sign in -front of it. A sample might be:<br> -<br> -*.* $mychannel<br> -<br> -In its current form, output channels primarily provide the ability to -size-limit an output file. To do so, specify a maximum size. When this -size is reached, rsyslogd will execute the action-on-max-size command -and then reopen the file and retry. The command should be something -like a <a href="log_rotation_fix_size.html">log rotation -script</a> or a similar thing.</p> -<p>If there is no action-on-max-size command or the command did -not resolve the situation, the file is closed and never reopened by -rsyslogd (except, of course, by huping it). This logic was integrated -when we first experienced severe issues with files larger 2gb, which -could lead to rsyslogd dumping core. In such cases, it is more -appropriate to stop writing to a single file. Meanwhile, rsyslogd has -been fixed to support files larger 2gb, but obviously only on file -systems and operating system versions that do so. So it can still make -sense to enforce a 2gb file size limit.</p> -<h2>Filter Conditions</h2> -<p>Rsyslog offers four different types "filter conditions":</p> -<ul> -<li>BSD-style blocks</li> -<li>"traditional" severity and facility based selectors</li> -<li>property-based filters</li> -<li>expression-based filters</li> -</ul> -<h3>Blocks</h3> -<p>Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each -block of lines is separated from the previous block by a program or -hostname specification. A block will only log messages corresponding to -the most recent program and hostname specifications given. Thus, a -block which selects ‘ppp’ as the program, directly followed by a block -that selects messages from the hostname ‘dialhost’, then the second -block will only log messages from the ppp program on dialhost. -</p> -<p>A program specification is a line beginning with ‘!prog’ and -the following blocks will be associated with calls to syslog from that -specific program. A program specification for ‘foo’ will also match any -message logged by the kernel with the prefix ‘foo: ’. Alternatively, a -program specification ‘-foo’ causes the following blocks to be applied -to messages from any program but the one specified. A hostname -specification of the form ‘+hostname’ and the following blocks will be -applied to messages received from the specified hostname. -Alternatively, a hostname specification ‘-hostname’ causes the -following blocks to be applied to messages from any host but the one -specified. If the hostname is given as ‘@’, the local hostname will be -used. (NOT YET IMPLEMENTED) A program or hostname specification may be -reset by giving the program or hostname as ‘*’.</p> -<p>Please note that the "#!prog", "#+hostname" and "#-hostname" -syntax available in BSD syslogd is not supported by rsyslogd. By -default, no hostname or program is set.</p> -<h3>Selectors</h3> -<p><b>Selectors are the traditional way of filtering syslog -messages.</b> They have been kept in rsyslog with their original -syntax, because it is well-known, highly effective and also needed for -compatibility with stock syslogd configuration files. If you just need -to filter based on priority and facility, you should do this with -selector lines. They are <b>not</b> second-class citizens -in rsyslog and offer the best performance for this job.</p> -<p>The selector field itself again consists of two parts, a -facility and a priority, separated by a period (".''). Both parts are -case insensitive and can also be specified as decimal numbers, but -don't do that, you have been warned. Both facilities and priorities are -described in rsyslog(3). The names mentioned below correspond to the -similar LOG_-values in /usr/include/rsyslog.h.<br> -<br> -The facility is one of the following keywords: auth, authpriv, cron, -daemon, kern, lpr, mail, mark, news, security (same as auth), syslog, -user, uucp and local0 through local7. The keyword security should not -be used anymore and mark is only for internal use and therefore should -not be used in applications. Anyway, you may want to specify and -redirect these messages here. The facility specifies the subsystem that -produced the message, i.e. all mail programs log with the mail facility -(LOG_MAIL) if they log using syslog.<br> -<br> -The priority is one of the following keywords, in ascending order: -debug, info, notice, warning, warn (same as warning), err, error (same -as err), crit, alert, emerg, panic (same as emerg). The keywords error, -warn and panic are deprecated and should not be used anymore. The -priority defines the severity of the message.<br> -<br> -The behavior of the original BSD syslogd is that all messages of the -specified priority and higher are logged according to the given action. -Rsyslogd behaves the same, but has some extensions.<br> -<br> -In addition to the above mentioned names the rsyslogd(8) understands -the following extensions: An asterisk ("*'') stands for all facilities -or all priorities, depending on where it is used (before or after the -period). The keyword none stands for no priority of the given facility.<br> -<br> -You can specify multiple facilities with the same priority pattern in -one statement using the comma (",'') operator. You may specify as much -facilities as you want. Remember that only the facility part from such -a statement is taken, a priority part would be skipped.</p> -<p>Multiple selectors may be specified for a single action using -the semicolon (";'') separator. Remember that each selector in the -selector field is capable to overwrite the preceding ones. Using this -behavior you can exclude some priorities from the pattern.</p> -<p>Rsyslogd has a syntax extension to the original BSD source, -that makes its use more intuitively. You may precede every priority -with an equation sign ("='') to specify only this single priority and -not any of the above. You may also (both is valid, too) precede the -priority with an exclamation mark ("!'') to ignore all that -priorities, either exact this one or this and any higher priority. If -you use both extensions than the exclamation mark must occur before the -equation sign, just use it intuitively.</p> -<h3>Property-Based Filters</h3> -<p>Property-based filters are unique to rsyslogd. They allow to -filter on any property, like HOSTNAME, syslogtag and msg. A list of all -currently-supported properties can be found in the <a href="property_replacer.html">property replacer documentation</a> -(but keep in mind that only the properties, not the replacer is -supported). With this filter, each properties can be checked against a -specified value, using a specified compare operation.</p> -<p>A property-based filter must start with a colon in column 0. -This tells rsyslogd that it is the new filter type. The colon must be -followed by the property name, a comma, the name of the compare -operation to carry out, another comma and then the value to compare -against. This value must be quoted. There can be spaces and tabs -between the commas. Property names and compare operations are -case-sensitive, so "msg" works, while "MSG" is an invalid property -name. In brief, the syntax is as follows:</p> -<p><code><b>:property, [!]compare-operation, "value"</b></code></p> -<p>The following <b>compare-operations</b> are -currently supported:</p> -<table id="table1" border="1" width="100%"> -<tbody> -<tr> -<td>contains</td> -<td>Checks if the string provided in value is contained in -the property. There must be an exact match, wildcards are not supported.</td> -</tr> -<tr> -<td>isequal</td> -<td>Compares the "value" string provided and the property -contents. These two values must be exactly equal to match. The -difference to contains is that contains searches for the value anywhere -inside the property value, whereas all characters must be identical for -isequal. As such, isequal is most useful for fields like syslogtag or -FROMHOST, where you probably know the exact contents.</td> -</tr> -<tr> -<td>startswith</td> -<td>Checks if the value is found exactly at the beginning -of the property value. For example, if you search for "val" with -<p><code><b>:msg, startswith, "val"</b></code></p> -<p>it will be a match if msg contains "values are in this -message" but it won't match if the msg contains "There are values in -this message" (in the later case, contains would match). Please note -that "startswith" is by far faster than regular expressions. So even -once they are implemented, it can make very much sense -(performance-wise) to use "startswith".</p> -</td> -</tr> -<tr> -<td>regex</td> -<td>Compares the property against the provided POSIX -regular -expression.</td> -</tr> -</tbody> -</table> -<p>You can use the bang-character (!) immediately in front of a -compare-operation, the outcome of this operation is negated. For -example, if msg contains "This is an informative message", the -following sample would not match:</p> -<p><code><b>:msg, contains, "error"</b></code></p> -<p>but this one matches:</p> -<p><code><b>:msg, !contains, "error"</b></code></p> -<p>Using negation can be useful if you would like to do some -generic processing but exclude some specific events. You can use the -discard action in conjunction with that. A sample would be:</p> -<p><code><b>*.* -/var/log/allmsgs-including-informational.log<br> -:msg, contains, "informational" <font color="#ff0000" size="4">~</font> -<br> -*.* /var/log/allmsgs-but-informational.log</b></code></p> -<p>Do not overlook the red tilde in line 2! In this sample, all -messages are written to the file allmsgs-including-informational.log. -Then, all messages containing the string "informational" are discarded. -That means the config file lines below the "discard line" (number 2 in -our sample) will not be applied to this message. Then, all remaining -lines will also be written to the file allmsgs-but-informational.log.</p> -<p><b>Value</b> is a quoted string. It supports some -escape sequences:</p> -<p>\" - the quote character (e.g. "String with \"Quotes\"")<br> -\\ - the backslash character (e.g. "C:\\tmp")</p> -<p>Escape sequences always start with a backslash. Additional -escape sequences might be added in the future. Backslash characters <b>must</b> -be escaped. Any other sequence then those outlined above is invalid and -may lead to unpredictable results.</p> -<p>Probably, "msg" is the most prominent use case of property -based filters. It is the actual message text. If you would like to -filter based on some message content (e.g. the presence of a specific -code), this can be done easily by:</p> -<p><code><b>:msg, contains, "ID-4711"</b></code></p> -<p>This filter will match when the message contains the string -"ID-4711". Please note that the comparison is case-sensitive, so it -would not match if "id-4711" would be contained in the message.</p> -<p><code><b>:msg, regex, "fatal .* error"</b></code></p> -<p>This filter uses a POSIX regular expression. It matches when -the -string contains the words "fatal" and "error" with anything in between -(e.g. "fatal net error" and "fatal lib error" but not "fatal error" as -two spaces are required by the regular expression!).</p> -<p>Getting property-based filters right can sometimes be -challenging. In order to help you do it with as minimal effort as -possible, rsyslogd spits out debug information for all property-based -filters during their evaluation. To enable this, run rsyslogd in -foreground and specify the "-d" option.</p> -<p>Boolean operations inside property based filters (like -'message contains "ID17" or message contains "ID18"') are currently not -supported (except for "not" as outlined above). Please note that while -it is possible to query facility and severity via property-based -filters, it is far more advisable to use classic selectors (see above) -for those cases.</p> -<h3>Expression-Based Filters</h3> -Expression based filters allow -filtering on arbitrary complex expressions, which can include boolean, -arithmetic and string operations. Expression filters will evolve into a -full configuration scripting language. Unfortunately, their syntax will -slightly change during that process. So if you use them now, you need -to be prepared to change your configuration files some time later. -However, we try to implement the scripting facility as soon as possible -(also in respect to stage work needed). So the window of exposure is -probably not too long.<br> -<br> -Expression based filters are indicated by the keyword "if" in column 1 -of a new line. They have this format:<br> -<br> -if expr then action-part-of-selector-line<br> -<br> -"If" and "then" are fixed keywords that mus be present. "expr" is a -(potentially quite complex) expression. So the <a href="expression.h">expression documentation</a> for -details. "action-part-of-selector-line" is an action, just as you know -it (e.g. "/var/log/logfile" to write to that file).<br> -<br> -A few quick samples:<br> -<br> -<code> -*.* /var/log/file1 # the traditional way<br> -if $msg contains 'error' /var/log/errlog # the expression-based way<br> -</code> -<br> -Right now, you need to specify numerical values if you would like to -check for facilities and severity. These can be found in <a href="http://www.ietf.org/rfc/rfc3164.txt">RFC 3164</a>. -If you don't like that, you can of course also use the textual property -- just be sure to use the right one. As expression support is enhanced, -this will change. For example, if you would like to filter on message -that have facility local0, start with "DEVNAME" and have either -"error1" or "error0" in their message content, you could use the -following filter:<br> -<br> -<code> -if $syslogfacility-text == 'local0' and $msg -startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains -'error0') then /var/log/somelog<br> -</code> -<br> -Please note that the above <span style="font-weight: bold;">must -all be on one line</span>! And if you would like to store all -messages except those that contain "error1" or "error0", you just need -to add a "not":<br> -<br> -<code> -if $syslogfacility-text == 'local0' and $msg -startswith 'DEVNAME' and <span style="font-weight: bold;">not</span> -($msg contains 'error1' or $msg contains -'error0') then /var/log/somelog<br> -</code> -<br> -If you would like to do case-insensitive comparisons, use -"contains_i" instead of "contains" and "startswith_i" instead of -"startswith".<br> -<br> -Note that regular expressions are currently NOT -supported in expression-based filters. These will be added later when -function support is added to the expression engine (the reason is that -regular expressions will be a separate loadable module, which requires -some more prequisites before it can be implemented).<br> -<h2>ACTIONS</h2> -<p>The action field of a rule describes what to do with the -message. In general, message content is written to a kind of "logfile". -But also other actions might be done, like writing to a database table -or forwarding to another host.<br> -<br> -Templates can be used with all actions. If used, the specified template -is used to generate the message content (instead of the default -template). To specify a template, write a semicolon after the action -value immediately followed by the template name.<br> -<br> -Beware: templates MUST be defined BEFORE they are used. It is OK to -define some templates, then use them in selector lines, define more -templates and use use them in the following selector lines. But it is -NOT permitted to use a template in a selector line that is above its -definition. If you do this, the action will be ignored.</p> -<p><b>You can have multiple actions for a single selector </b> (or -more precisely a single filter of such a selector line). Each action -must be on its own line and the line must start with an ampersand -('&') character and have no filters. An example would be</p> -<p><code><b>*.=crit rger<br> -& root<br> -& /var/log/critmsgs</b></code></p> -<p>These three lines send critical messages to the user rger and -root and also store them in /var/log/critmsgs. <b>Using multiple -actions per selector is</b> convenient and also <b>offers -a performance benefit</b>. As the filter needs to be evaluated -only once, there is less computation required to process the directive -compared to the otherwise-equal config directives below:</p> -<p><code><b>*.=crit rger<br> -*.=crit root<br> -*.=crit /var/log/critmsgs</b></code></p> -<p> </p> -<h3>Regular File</h3> -<p>Typically messages are logged to real files. The file has to -be specified with full pathname, beginning with a slash "/''.<br> -<br> -You may prefix each entry with the minus "-'' sign to omit syncing the -file after every logging. Note that you might lose information if the -system crashes right behind a write attempt. Nevertheless this might -give you back some performance, especially if you run programs that use -logging in a very verbose manner.</p> -<p>If your system is connected to a reliable UPS and you receive -lots of log data (e.g. firewall logs), it might be a very good idea to -turn of -syncing by specifying the "-" in front of the file name. </p> -<p><b>The filename can be either static </b>(always -the same) or <b>dynamic</b> (different based on message -received). The later is useful if you would automatically split -messages into different files based on some message criteria. For -example, dynamic file name selectors allow you to split messages into -different files based on the host that sent them. With dynamic file -names, everything is automatic and you do not need any filters. </p> -<p>It works via the template system. First, you define a template -for the file name. An example can be seen above in the description of -template. We will use the "DynFile" template defined there. Dynamic -filenames are indicated by specifying a questions mark "?" instead of a -slash, followed by the template name. Thus, the selector line for our -dynamic file name would look as follows:</p> -<blockquote> -<code>*.* ?DynFile</code> -</blockquote> -<p>That's all you need to do. Rsyslog will now automatically -generate file names for you and store the right messages into the right -files. Please note that the minus sign also works with dynamic file -name selectors. Thus, to avoid syncing, you may use</p> -<blockquote> -<code>*.* -?DynFile</code></blockquote> -<p>And of course you can use templates to specify the output -format:</p> -<blockquote> -<code>*.* ?DynFile;MyTemplate</code></blockquote> -<p><b>A word of caution:</b> rsyslog creates files as -needed. So if a new host is using your syslog server, rsyslog will -automatically create a new file for it.</p> -<p><b>Creating directories is also supported</b>. For -example you can use the hostname as directory and the program name as -file name:</p> -<blockquote> -<code>$template DynFile,"/var/log/%HOSTNAME%/%programname%.log"</code></blockquote> -<h3>Named Pipes</h3> -<p>This version of rsyslogd(8) has support for logging output to -named pipes (fifos). A fifo or named pipe can be used as a destination -for log messages by prepending a pipe symbol ("|'') to the name of the -file. This is handy for debugging. Note that the fifo must be created -with the mkfifo(1) command before rsyslogd(8) is started.</p> -<h3>Terminal and Console</h3> -<p>If the file you specified is a tty, special tty-handling is -done, same with /dev/console.</p> -<h3>Remote Machine</h3> -<p>Rsyslogd provides full remote logging, i.e. is able to send -messages to a remote host running rsyslogd(8) and to receive messages -from remote hosts. Using this feature you're able to control all syslog -messages on one host, if all other machines will log remotely to that. -This tears down<br> -administration needs.<br> -<br> -<b>Please note that this version of rsyslogd by default does NOT -forward messages it has received from the network to another host. -Specify the "-h" option to enable this.</b></p> -<p>To forward messages to another host, prepend the hostname with -the at sign ("@"). A single at sign means that messages will -be forwarded via UDP protocol (the standard for syslog). If you prepend -two at signs ("@@"), the messages will be transmitted via TCP. Please -note that plain TCP based syslog is not officially standardized, but -most major syslogds support it (e.g. syslog-ng or WinSyslog). The -forwarding action indicator (at-sign) can be followed by one or more -options. If they are given, they must be immediately (without a space) -following the final at sign and be enclosed in parenthesis. The -individual options must be separated by commas. The following options -are right now defined:</p> -<table id="table2" border="1" width="100%"> -<tbody> -<tr> -<td> -<p align="center"><b>z<number></b></p> -</td> -<td>Enable zlib-compression for the message. The -<number> is the compression level. It can be 1 (lowest -gain, lowest CPU overhead) to 9 (maximum compression, highest CPU -overhead). The level can also be 0, which means "no compression". If -given, the "z" option is ignored. So this does not make an awful lot of -sense. There is hardly a difference between level 1 and 9 for typical -syslog messages. You can expect a compression gain between 0% and 30% -for typical messages. Very chatty messages may compress up to 50%, but -this is seldom seen with typically traffic. Please note that rsyslogd -checks the compression gain. Messages with 60 bytes or less will never -be compressed. This is because compression gain is pretty unlikely and -we prefer to save CPU cycles. Messages over that size are always -compressed. However, it is checked if there is a gain in compression -and only if there is, the compressed message is transmitted. Otherwise, -the uncompressed messages is transmitted. This saves the receiver CPU -cycles for decompression. It also prevents small message to actually -become larger in compressed form. -<p><b>Please note that when a TCP transport is used, -compression will also turn on syslog-transport-tls framing. See the "o" -option for important information on the implications.</b></p> -<p>Compressed messages are automatically detected and -decompressed by the receiver. There is nothing that needs to be -configured on the receiver side.</p> -</td> -</tr> -<tr> -<td> -<p align="center"><b>o</b></p> -</td> -<td><b>This option is experimental. Use at your own -risk and only if you know why you need it! If in doubt, do NOT turn it -on.</b> -<p>This option is only valid for plain TCP based -transports. It selects a different framing based on IETF internet draft -syslog-transport-tls-06. This framing offers some benefits over -traditional LF-based framing. However, the standardization effort is -not yet complete. There may be changes in upcoming versions of this -standard. Rsyslog will be kept in line with the standard. There is some -chance that upcoming changes will be incompatible to the current -specification. In this case, all systems using -transport-tls framing -must be upgraded. There will be no effort made to retain compatibility -between different versions of rsyslog. The primary reason for that is -that it seems technically impossible to provide compatibility between -some of those changes. So you should take this note very serious. It is -not something we do not *like* to do (and may change our mind if enough -people beg...), it is something we most probably *can not* do for -technical reasons (aka: you can beg as much as you like, it won't -change anything...).</p> -<p>The most important implication is that compressed syslog -messages via TCP must be considered with care. Unfortunately, it is -technically impossible to transfer compressed records over traditional -syslog plain tcp transports, so you are left with two evil choices...</p> -</td> -</tr> -</tbody> -</table> -<p><br> -The hostname may be followed by a colon and the destination port.</p> -<p>The following is an example selector line with forwarding:</p> -<p>*.* @@(o,z9)192.168.0.1:1470</p> -<p>In this example, messages are forwarded via plain TCP with -experimental framing and maximum compression to the host 192.168.0.1 at -port 1470.</p> -<p>*.* @192.168.0.1</p> -<p>In the example above, messages are forwarded via UDP to the -machine 192.168.0.1, the destination port defaults to 514. Messages -will not be compressed.</p> -<p>Note that IPv6 addresses contain colons. So if an IPv6 address is specified -in the hostname part, rsyslogd could not detect where the IP address ends -and where the port starts. There is a syntax extension to support this: -put squary brackets around the address (e.g. "[2001::1]"). Square -brackets also work with real host names and IPv4 addresses, too. -<p>A valid sample to send messages to the IPv6 host 2001::1 at port 515 -is as follows: -<p>*.* @[2001::1]:515 -<p>This works with TCP, too. -<p><b>Note to sysklogd users:</b> sysklogd does <b>not</b> -support RFC 3164 format, which is the default forwarding template in -rsyslog. As such, you will experience duplicate hostnames if rsyslog is -the sender and sysklogd is the receiver. The fix is simple: you need to -use a different template. Use that one:</p> -<p class="MsoPlainText">$template -sysklogd,"<%PRI%>%TIMESTAMP% %syslogtag%%msg%\""<br> -*.* @192.168.0.1;sysklogd</p> -<h3>List of Users</h3> -<p>Usually critical messages are also directed to "root'' on -that machine. You can specify a list of users that shall get the -message by simply writing the login. You may specify more than one user -by separating them with commas (",''). If they're logged in they get -the message. Don't think a mail would be sent, that might be too late.</p> -<h3>Everyone logged on</h3> -<p>Emergency messages often go to all users currently online to -notify them that something strange is happening with the system. To -specify this wall(1)-feature use an asterisk ("*'').</p> -<h3>Call Plugin</h3> -<p>This is a generic way to call an output plugin. The plugin -must support this functionality. Actual parameters depend on the -module, so see the module's doc on what to supply. The general syntax -is as follows:</p> -<p>:modname:params;template</p> -<p>Currently, the ommysql database output module supports this -syntax (in addtion to the ">" syntax it traditionally -supported). For ommysql, the module name is "ommysql" and the params -are the traditional ones. The ;template part is not module specific, it -is generic rsyslog functionality available to all modules.</p> -<p>As an example, the ommysql module may be called as follows:</p> -<p>:ommysql:dbhost,dbname,dbuser,dbpassword;dbtemplate</p> -<p>For details, please see the "Database Table" section of this -documentation.</p> -<p>Note: as of this writing, the ":modname:" part is hardcoded -into the module. So the name to use is not necessarily the name the -module's plugin file is called.</p> -<h3>Database Table</h3> -<p>This allows logging of the message to a database table. -Currently, only MySQL databases are supported. However, other database -drivers will most probably be developed as plugins. By default, a <a href="http://www.monitorware.com/">MonitorWare</a>-compatible -schema is required for this to work. You can create that schema with -the createDB.SQL file that came with the rsyslog package. You can also<br> -use any other schema of your liking - you just need to define a proper -template and assign this template to the action.<br> -<br> -The database writer is called by specifying a greater-then sign -(">") in front of the database connect information. Immediately -after that<br> -sign the database host name must be given, a comma, the database name, -another comma, the database user, a comma and then the user's password. -If a specific template is to be used, a semicolon followed by the -template name can follow the connect information. This is as follows:<br> -<br> ->dbhost,dbname,dbuser,dbpassword;dbtemplate</p> -<p><b>Important: to use the database functionality, the -MySQL output module must be loaded in the config file</b> BEFORE -the first database table action is used. This is done by placing the</p> -<p><code><b>$ModLoad ommysql</b></code></p> -<p>directive some place above the first use of the database write -(we recommend doing at the the beginning of the config file).</p> -<h3>Discard</h3> -<p>If the discard action is carried out, the received message is -immediately discarded. No further processing of it occurs. Discard has -primarily been added to filter out messages before carrying on any -further processing. For obvious reasons, the results of "discard" are -depending on where in the configuration file it is being used. Please -note that once a message has been discarded there is no way to retrieve -it in later configuration file lines.</p> -<p>Discard can be highly effective if you want to filter out some -annoying messages that otherwise would fill your log files. To do that, -place the discard actions early in your log files. This often plays -well with property-based filters, giving you great freedom in -specifying what you do not want.</p> -<p>Discard is just the single tilde character with no further -parameters:</p> -<p>~</p> -<p>For example,</p> -<p>*.* ~</p> -<p>discards everything (ok, you can achive the same by not -running rsyslogd at all...).</p> -<h3>Output Channel</h3> -<p>Binds an output channel definition (see there for details) to -this action. Output channel actions must start with a $-sign, e.g. if -you would like to bind your output channel definition "mychannel" to -the action, use "$mychannel". Output channels support template -definitions like all all other actions.</p> -<h3>Shell Execute</h3> -<p>This executes a program in a subshell. The program is passed -the template-generated message as the only command line parameter. -Rsyslog waits until the program terminates and only then continues to -run.</p> -<p>^program-to-execute;template</p> -<p>The program-to-execute can be any valid executable. It -receives the template string as a single parameter (argv[1]).</p> -<p><b>WARNING:</b> The Shell Execute action was added -to serve an urgent need. While it is considered reasonable save when -used with some thinking, its implications must be considered. The -current implementation uses a system() call to execute the command. -This is not the best way to do it (and will hopefully changed in -further releases). Also, proper escaping of special characters is done -to prevent command injection. However, attackers always find smart ways -to circumvent escaping, so we can not say if the escaping applied will -really safe you from all hassles. Lastly, rsyslog will wait until the -shell command terminates. Thus, a program error in it (e.g. an infinite -loop) can actually disable rsyslog. Even without that, during the -programs run-time no messages are processed by rsyslog. As the IP -stacks buffers are quickly overflowed, this bears an increased risk of -message loss. You must be aware of these implications. Even though they -are severe, there are several cases where the "shell execute" action is -very useful. This is the reason why we have included it in its current -form. To mitigate its risks, always a) test your program thoroughly, b) -make sure its runtime is as short as possible (if it requires a longer -run-time, you might want to spawn your own sub-shell asynchronously), -c) apply proper firewalling so that only known senders can send syslog -messages to rsyslog. Point c) is especially important: if rsyslog is -accepting message from any hosts, chances are much higher that an -attacker might try to exploit the "shell execute" action.</p> -<h2>TEMPLATE NAME</h2> -<p>Every ACTION can be followed by a template name. If so, that -template is used for message formatting. If no name is given, a -hard-coded default template is used for the action. There can only be -one template name for each given action. The default template is -specific to each action. For a description of what a template is and -what you can do with it, see "TEMPLATES" at the top of this document.</p> -<h2>EXAMPLES</h2> -<p>Below are example for templates and selector lines. I hope +<h2><a href="rsyslog_conf_templates.html">Templates</a></h2> +<h2><a href="rsyslog_conf_output.html">Output Channels</a></h2> +<h2><a href="rsyslog_conf_filter.html">Filter Conditions</a></h2> +<h2><a href="rsyslog_conf_actions.html">Actions</a></h2> +<h2><a href="rsyslog_conf_examples.html">Examples</a></h2> +<p>Here you will find examples for templates and selector lines. I hope they are self-explanatory. If not, please see www.monitorware.com/rsyslog/ for advise.</p> -<h3>TEMPLATES</h3> -<p>Please note that the samples are split across multiple lines. -A template MUST NOT actually be split across multiple lines.<br> -<br> -A template that resembles traditional syslogd file output:<br> -$template TraditionalFormat,"%timegenerated% %HOSTNAME%<br> -%syslogtag%%msg:::drop-last-lf%\n"<br> -<br> -A template that tells you a little more about the message:<br> -$template -precise,"%syslogpriority%,%syslogfacility%,%timegenerated%,%HOSTNAME%,<br> -%syslogtag%,%msg%\n"<br> -<br> -A template for RFC 3164 format:<br> -$template RFC3164fmt,"<%PRI%>%TIMESTAMP% %HOSTNAME% -%syslogtag%%msg%"<br> -<br> -A template for the format traditonally used for user messages:<br> -$template usermsg," XXXX%syslogtag%%msg%\n\r"<br> -<br> -And a template with the traditonal wall-message format:<br> -$template wallmsg,"\r\n\7Message from syslogd@%HOSTNAME% at -%timegenerated%<br> -<br> -A template that can be used for the database write (please note the SQL<br> -template option)<br> -$template MySQLInsert,"insert iut, message, receivedat values<br> -('%iut%', '%msg:::UPPERCASE%', '%timegenerated:::date-mysql%')<br> -into systemevents\r\n", SQL<br> -<br> -The following template emulates <a href="http://www.winsyslog.com/en/">WinSyslog</a> -format (it's an <a href="http://www.adiscon.com/en/">Adiscon</a> -format, you do not feel bad if you don't know it ;)). It's interesting -to see how it takes different parts out of the date stamps. What -happens is that the date stamp is split into the actual date and time -and the these two are combined with just a comma in between them.<br> -<br> -$template WinSyslogFmt,"%HOSTNAME%,%timegenerated:1:10:date-rfc3339%,<br> -%timegenerated:12:19:date-rfc3339%,%timegenerated:1:10:date-rfc3339%,<br> -%timegenerated:12:19:date-rfc3339%,%syslogfacility%,%syslogpriority%,<br> -%syslogtag%%msg%\n"</p> -<h3>SELECTOR LINES</h3> -<p># Store critical stuff in critical<br> -#<br> -*.=crit;kern.none /var/adm/critical<br> -<br> -This will store all messages with the priority crit in the file -/var/adm/critical, except for any kernel message.<br> -<br> -<br> -# Kernel messages are first, stored in the kernel<br> -# file, critical messages and higher ones also go<br> -# to another host and to the console. Messages to<br> -# the host finlandia are forwarded in RFC 3164<br> -# format (using the template defined above).<br> -#<br> -kern.* /var/adm/kernel<br> -kern.crit @finlandia;RFC3164fmt<br> -kern.crit /dev/console<br> -kern.info;kern.!err /var/adm/kernel-info<br> -<br> -The first rule direct any message that has the kernel facility to the -file /var/adm/kernel.<br> -<br> -The second statement directs all kernel messages of the priority crit -and higher to the remote host finlandia. This is useful, because if the -host crashes and the disks get irreparable errors you might not be able -to read the stored messages. If they're on a remote host, too, you -still can try to find out the reason for the crash.<br> -<br> -The third rule directs these messages to the actual console, so the -person who works on the machine will get them, too.<br> -<br> -The fourth line tells rsyslogd to save all kernel messages that come -with priorities from info up to warning in the file -/var/adm/kernel-info. Everything from err and higher is excluded.<br> -<br> -<br> -# The tcp wrapper loggs with mail.info, we display<br> -# all the connections on tty12<br> -#<br> -mail.=info /dev/tty12<br> -<br> -This directs all messages that uses mail.info (in source LOG_MAIL | -LOG_INFO) to /dev/tty12, the 12th console. For example the tcpwrapper -tcpd(8) uses this as it's default.<br> -<br> -<br> -# Store all mail concerning stuff in a file<br> -#<br> -mail.*;mail.!=info /var/adm/mail<br> -<br> -This pattern matches all messages that come with the mail facility, -except for the info priority. These will be stored in the file -/var/adm/mail.<br> -<br> -<br> -# Log all mail.info and news.info messages to info<br> -#<br> -mail,news.=info /var/adm/info<br> -<br> -This will extract all messages that come either with mail.info or with -news.info and store them in the file /var/adm/info.<br> -<br> -<br> -# Log info and notice messages to messages file<br> -#<br> -*.=info;*.=notice;\<br> -mail.none /var/log/messages<br> -<br> -This lets rsyslogd log all messages that come with either the info or -the notice facility into the file /var/log/messages, except for all<br> -messages that use the mail facility.<br> -<br> -<br> -# Log info messages to messages file<br> -#<br> -*.=info;\<br> -mail,news.none /var/log/messages<br> -<br> -This statement causes rsyslogd to log all messages that come with the -info priority to the file /var/log/messages. But any message coming -either with the mail or the news facility will not be stored.<br> -<br> -<br> -# Emergency messages will be displayed using wall<br> -#<br> -*.=emerg *<br> -<br> -This rule tells rsyslogd to write all emergency messages to all -currently logged in users. This is the wall action.<br> -<br> -<br> -# Messages of the priority alert will be directed<br> -# to the operator<br> -#<br> -*.alert root,rgerhards<br> -<br> -This rule directs all messages with a priority of alert or higher to -the terminals of the operator, i.e. of the users "root'' and -"rgerhards'' if they're logged in.<br> -<br> -<br> -*.* @finlandia<br> -<br> -This rule would redirect all messages to a remote host called -finlandia. This is useful especially in a cluster of machines where all -syslog messages will be stored on only one machine.<br> -<br> -In the format shown above, UDP is used for transmitting the message. -The destination port is set to the default auf 514. Rsyslog is also -capable of using much more secure and reliable TCP sessions for message -forwarding. Also, the destination port can be specified. To select TCP, -simply add one additional @ in front of the host name (that is, @host -is UPD, @@host is TCP). For example:<br> -<br> -<br> -*.* @@finlandia<br> -<br> -To specify the destination port on the remote machine, use a colon -followed by the port number after the machine name. The following -forwards to port 1514 on finlandia:<br> -<br> -<br> -*.* @@finlandia:1514<br> -<br> -This syntax works both with TCP and UDP based syslog. However, you will -probably primarily need it for TCP, as there is no well-accepted port -for this transport (it is non-standard). For UDP, you can usually stick -with the default auf 514, but might want to modify it for security rea-<br> -sons. If you would like to do that, it's quite easy:<br> -<br> -<br> -*.* @finlandia:1514<br> -<br> -<br> -<br> -*.* >dbhost,dbname,dbuser,dbpassword;dbtemplate<br> -<br> -This rule writes all message to the database "dbname" hosted on -"dbhost". The login is done with user "dbuser" and password -"dbpassword". The actual table that is updated is specified within the -template (which contains the insert statement). The template is called -"dbtemplate" in this case.</p> -<p>:msg,contains,"error" @errorServer</p> -<p>This rule forwards all messages that contain the word "error" -in the msg part to the server "errorServer". Forwarding is via UDP. -Please note the colon in fron</p> -<h2>CONFIGURATION FILE SYNTAX DIFFERENCES</h2> +<h2>Configuration File Syntax Differences</h2> <p>Rsyslogd uses a slightly different syntax for its configuration file than the original BSD sources. Originally all messages of a specific priority and above were forwarded to the log @@ -1258,4 +68,15 @@ additional <a href="features.html">features</a> (like template and database support). For obvious reasons, the syntax for defining such features is available in rsyslogd, only.</p> -</body></html> + +<p>[<a href="rsyslog_conf.html">back to top</a>] +[<a href="manual.html">manual index</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008,2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> +> diff --git a/doc/rsyslog_conf_actions.html b/doc/rsyslog_conf_actions.html new file mode 100644 index 00000000..2ef3f4b0 --- /dev/null +++ b/doc/rsyslog_conf_actions.html @@ -0,0 +1,336 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Actions - rsyslog.conf</title></head> +<body> +<p>This is a part of the rsyslog.conf documentation.</p> +<a href="rsyslog_conf.html">back</a> +<h2>Actions</h2> +<p>The action field of a rule describes what to do with the +message. In general, message content is written to a kind of "logfile". +But also other actions might be done, like writing to a database table +or forwarding to another host.<br> +<br> +Templates can be used with all actions. If used, the specified template +is used to generate the message content (instead of the default +template). To specify a template, write a semicolon after the action +value immediately followed by the template name.<br> +<br> +Beware: templates MUST be defined BEFORE they are used. It is OK to +define some templates, then use them in selector lines, define more +templates and use use them in the following selector lines. But it is +NOT permitted to use a template in a selector line that is above its +definition. If you do this, the action will be ignored.</p> +<p><b>You can have multiple actions for a single selector </b> (or +more precisely a single filter of such a selector line). Each action +must be on its own line and the line must start with an ampersand +('&') character and have no filters. An example would be</p> +<p><code><b>*.=crit rger<br> +& root<br> +& /var/log/critmsgs</b></code></p> +<p>These three lines send critical messages to the user rger and +root and also store them in /var/log/critmsgs. <b>Using multiple +actions per selector is</b> convenient and also <b>offers +a performance benefit</b>. As the filter needs to be evaluated +only once, there is less computation required to process the directive +compared to the otherwise-equal config directives below:</p> +<p><code><b>*.=crit rger<br> +*.=crit root<br> +*.=crit /var/log/critmsgs</b></code></p> +<p> </p> +<h3>Regular File</h3> +<p>Typically messages are logged to real files. The file has to +be specified with full pathname, beginning with a slash "/''.<br> +<br> +<br> +You may prefix each entry with the minus "-'' sign to omit syncing the +file after every logging. Note that you might lose information if the +system crashes right behind a write attempt. Nevertheless this might +give you back some performance, especially if you run programs that use +logging in a very verbose manner.</p> +<p>If your system is connected to a reliable UPS and you receive +lots of log data (e.g. firewall logs), it might be a very good idea to +turn of +syncing by specifying the "-" in front of the file name. </p> +<p><b>The filename can be either static </b>(always +the same) or <b>dynamic</b> (different based on message +received). The later is useful if you would automatically split +messages into different files based on some message criteria. For +example, dynamic file name selectors allow you to split messages into +different files based on the host that sent them. With dynamic file +names, everything is automatic and you do not need any filters. </p> +<p>It works via the template system. First, you define a template +for the file name. An example can be seen above in the description of +template. We will use the "DynFile" template defined there. Dynamic +filenames are indicated by specifying a questions mark "?" instead of a +slash, followed by the template name. Thus, the selector line for our +dynamic file name would look as follows:</p> +<blockquote> +<code>*.* ?DynFile</code> +</blockquote> +<p>That's all you need to do. Rsyslog will now automatically +generate file names for you and store the right messages into the right +files. Please note that the minus sign also works with dynamic file +name selectors. Thus, to avoid syncing, you may use</p> +<blockquote> +<code>*.* -?DynFile</code></blockquote> +<p>And of course you can use templates to specify the output +format:</p> +<blockquote> +<code>*.* ?DynFile;MyTemplate</code></blockquote> +<p><b>A word of caution:</b> rsyslog creates files as +needed. So if a new host is using your syslog server, rsyslog will +automatically create a new file for it.</p> +<p><b>Creating directories is also supported</b>. For +example you can use the hostname as directory and the program name as +file name:</p> +<blockquote> +<code>$template DynFile,"/var/log/%HOSTNAME%/%programname%.log"</code></blockquote> +<h3>Named Pipes</h3> +<p>This version of rsyslogd(8) has support for logging output to +named pipes (fifos). A fifo or named pipe can be used as a destination +for log messages by prepending a pipe symbol ("|'') to the name of the +file. This is handy for debugging. Note that the fifo must be created +with the mkfifo(1) command before rsyslogd(8) is started.</p> +<h3>Terminal and Console</h3> +<p>If the file you specified is a tty, special tty-handling is +done, same with /dev/console.</p> +<h3>Remote Machine</h3> +<p>Rsyslogd provides full remote logging, i.e. is able to send +messages to a remote host running rsyslogd(8) and to receive messages +from remote hosts. Using this feature you're able to control all syslog +messages on one host, if all other machines will log remotely to that. +This tears down<br> +administration needs.<br> +<br> +<b>Please note that this version of rsyslogd by default does NOT +forward messages it has received from the network to another host. +Specify the "-h" option to enable this.</b></p> +<p>To forward messages to another host, prepend the hostname with +the at sign ("@"). A single at sign means that messages will +be forwarded via UDP protocol (the standard for syslog). If you prepend +two at signs ("@@"), the messages will be transmitted via TCP. Please +note that plain TCP based syslog is not officially standardized, but +most major syslogds support it (e.g. syslog-ng or WinSyslog). The +forwarding action indicator (at-sign) can be followed by one or more +options. If they are given, they must be immediately (without a space) +following the final at sign and be enclosed in parenthesis. The +individual options must be separated by commas. The following options +are right now defined:</p> +<table id="table2" border="1" width="100%"> +<tbody> +<tr> +<td> +<p align="center"><b>z<number></b></p> +</td> +<td>Enable zlib-compression for the message. The +<number> is the compression level. It can be 1 (lowest +gain, lowest CPU overhead) to 9 (maximum compression, highest CPU +overhead). The level can also be 0, which means "no compression". If +given, the "z" option is ignored. So this does not make an awful lot of +sense. There is hardly a difference between level 1 and 9 for typical +syslog messages. You can expect a compression gain between 0% and 30% +for typical messages. Very chatty messages may compress up to 50%, but +this is seldom seen with typically traffic. Please note that rsyslogd +checks the compression gain. Messages with 60 bytes or less will never +be compressed. This is because compression gain is pretty unlikely and +we prefer to save CPU cycles. Messages over that size are always +compressed. However, it is checked if there is a gain in compression +and only if there is, the compressed message is transmitted. Otherwise, +the uncompressed messages is transmitted. This saves the receiver CPU +cycles for decompression. It also prevents small message to actually +become larger in compressed form. +<p><b>Please note that when a TCP transport is used, +compression will also turn on syslog-transport-tls framing. See the "o" +option for important information on the implications.</b></p> +<p>Compressed messages are automatically detected and +decompressed by the receiver. There is nothing that needs to be +configured on the receiver side.</p> +</td> +</tr> +<tr> +<td> +<p align="center"><b>o</b></p> +</td> +<td><b>This option is experimental. Use at your own +risk and only if you know why you need it! If in doubt, do NOT turn it +on.</b> +<p>This option is only valid for plain TCP based +transports. It selects a different framing based on IETF internet draft +syslog-transport-tls-06. This framing offers some benefits over +traditional LF-based framing. However, the standardization effort is +not yet complete. There may be changes in upcoming versions of this +standard. Rsyslog will be kept in line with the standard. There is some +chance that upcoming changes will be incompatible to the current +specification. In this case, all systems using -transport-tls framing +must be upgraded. There will be no effort made to retain compatibility +between different versions of rsyslog. The primary reason for that is +that it seems technically impossible to provide compatibility between +some of those changes. So you should take this note very serious. It is +not something we do not *like* to do (and may change our mind if enough +people beg...), it is something we most probably *can not* do for +technical reasons (aka: you can beg as much as you like, it won't +change anything...).</p> +<p>The most important implication is that compressed syslog +messages via TCP must be considered with care. Unfortunately, it is +technically impossible to transfer compressed records over traditional +syslog plain tcp transports, so you are left with two evil choices...</p> +</td> +</tr> +</tbody> +</table> +<p><br> +The hostname may be followed by a colon and the destination port.</p> +<p>The following is an example selector line with forwarding:</p> +<p>*.* @@(o,z9)192.168.0.1:1470</p> +<p>In this example, messages are forwarded via plain TCP with +experimental framing and maximum compression to the host 192.168.0.1 at +port 1470.</p> +<p>*.* @192.168.0.1</p> +<p>In the example above, messages are forwarded via UDP to the +machine 192.168.0.1, the destination port defaults to 514. Messages +will not be compressed.</p> +<p>Note that IPv6 addresses contain colons. So if an IPv6 address is specified +in the hostname part, rsyslogd could not detect where the IP address ends +and where the port starts. There is a syntax extension to support this: +put squary brackets around the address (e.g. "[2001::1]"). Square +brackets also work with real host names and IPv4 addresses, too. +<p>A valid sample to send messages to the IPv6 host 2001::1 at port 515 +is as follows: +<p>*.* @[2001::1]:515 +<p>This works with TCP, too. +<p><b>Note to sysklogd users:</b> sysklogd does <b>not</b> +support RFC 3164 format, which is the default forwarding template in +rsyslog. As such, you will experience duplicate hostnames if rsyslog is +the sender and sysklogd is the receiver. The fix is simple: you need to +use a different template. Use that one:</p> +<p class="MsoPlainText">$template +sysklogd,"<%PRI%>%TIMESTAMP% %syslogtag%%msg%\""<br> +*.* @192.168.0.1;sysklogd</p> +<h3>List of Users</h3> +<p>Usually critical messages are also directed to "root'' on +that machine. You can specify a list of users that shall get the +message by simply writing the login. You may specify more than one user +by separating them with commas (",''). If they're logged in they get +the message. Don't think a mail would be sent, that might be too late.</p> +<h3>Everyone logged on</h3> +<p>Emergency messages often go to all users currently online to +notify them that something strange is happening with the system. To +specify this wall(1)-feature use an asterisk ("*'').</p> +<h3>Call Plugin</h3> +<p>This is a generic way to call an output plugin. The plugin +must support this functionality. Actual parameters depend on the +module, so see the module's doc on what to supply. The general syntax +is as follows:</p> +<p>:modname:params;template</p> +<p>Currently, the ommysql database output module supports this +syntax (in addtion to the ">" syntax it traditionally +supported). For ommysql, the module name is "ommysql" and the params +are the traditional ones. The ;template part is not module specific, it +is generic rsyslog functionality available to all modules.</p> +<p>As an example, the ommysql module may be called as follows:</p> +<p>:ommysql:dbhost,dbname,dbuser,dbpassword;dbtemplate</p> +<p>For details, please see the "Database Table" section of this +documentation.</p> +<p>Note: as of this writing, the ":modname:" part is hardcoded +into the module. So the name to use is not necessarily the name the +module's plugin file is called.</p> +<h3>Database Table</h3> +<p>This allows logging of the message to a database table. +Currently, only MySQL databases are supported. However, other database +drivers will most probably be developed as plugins. By default, a <a href="http://www.monitorware.com/">MonitorWare</a>-compatible +schema is required for this to work. You can create that schema with +the createDB.SQL file that came with the rsyslog package. You can also<br> +use any other schema of your liking - you just need to define a proper +template and assign this template to the action.<br> +<br> +The database writer is called by specifying a greater-then sign +(">") in front of the database connect information. Immediately +after that<br> +sign the database host name must be given, a comma, the database name, +another comma, the database user, a comma and then the user's password. +If a specific template is to be used, a semicolon followed by the +template name can follow the connect information. This is as follows:<br> +<br> +>dbhost,dbname,dbuser,dbpassword;dbtemplate</p> +<p><b>Important: to use the database functionality, the +MySQL output module must be loaded in the config file</b> BEFORE +the first database table action is used. This is done by placing the</p> +<p><code><b>$ModLoad ommysql</b></code></p> +<p>directive some place above the first use of the database write +(we recommend doing at the the beginning of the config file).</p> +<h3>Discard</h3> +<p>If the discard action is carried out, the received message is +immediately discarded. No further processing of it occurs. Discard has +primarily been added to filter out messages before carrying on any +further processing. For obvious reasons, the results of "discard" are +depending on where in the configuration file it is being used. Please +note that once a message has been discarded there is no way to retrieve +it in later configuration file lines.</p> +<p>Discard can be highly effective if you want to filter out some +annoying messages that otherwise would fill your log files. To do that, +place the discard actions early in your log files. This often plays +well with property-based filters, giving you great freedom in +specifying what you do not want.</p> +<p>Discard is just the single tilde character with no further +parameters:</p> +<p>~</p> +<p>For example,</p> +<p>*.* ~</p> +<p>discards everything (ok, you can achive the same by not +running rsyslogd at all...).</p> +<h3>Output Channel</h3> +<p>Binds an output channel definition (see there for details) to +this action. Output channel actions must start with a $-sign, e.g. if +you would like to bind your output channel definition "mychannel" to +the action, use "$mychannel". Output channels support template +definitions like all all other actions.</p> +<h3>Shell Execute</h3> +<p>This executes a program in a subshell. The program is passed +the template-generated message as the only command line parameter. +Rsyslog waits until the program terminates and only then continues to +run.</p> +<p>^program-to-execute;template</p> +<p>The program-to-execute can be any valid executable. It +receives the template string as a single parameter (argv[1]).</p> +<p><b>WARNING:</b> The Shell Execute action was added +to serve an urgent need. While it is considered reasonable save when +used with some thinking, its implications must be considered. The +current implementation uses a system() call to execute the command. +This is not the best way to do it (and will hopefully changed in +further releases). Also, proper escaping of special characters is done +to prevent command injection. However, attackers always find smart ways +to circumvent escaping, so we can not say if the escaping applied will +really safe you from all hassles. Lastly, rsyslog will wait until the +shell command terminates. Thus, a program error in it (e.g. an infinite +loop) can actually disable rsyslog. Even without that, during the +programs run-time no messages are processed by rsyslog. As the IP +stacks buffers are quickly overflowed, this bears an increased risk of +message loss. You must be aware of these implications. Even though they +are severe, there are several cases where the "shell execute" action is +very useful. This is the reason why we have included it in its current +form. To mitigate its risks, always a) test your program thoroughly, b) +make sure its runtime is as short as possible (if it requires a longer +run-time, you might want to spawn your own sub-shell asynchronously), +c) apply proper firewalling so that only known senders can send syslog +messages to rsyslog. Point c) is especially important: if rsyslog is +accepting message from any hosts, chances are much higher that an +attacker might try to exploit the "shell execute" action.</p> +<h3>Template Name</h3> +<p>Every ACTION can be followed by a template name. If so, that +template is used for message formatting. If no name is given, a +hard-coded default template is used for the action. There can only be +one template name for each given action. The default template is +specific to each action. For a description of what a template is and +what you can do with it, see "TEMPLATES" at the top of this document.</p> + + +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> + diff --git a/doc/rsyslog_conf_examples.html b/doc/rsyslog_conf_examples.html new file mode 100644 index 00000000..b46460e5 --- /dev/null +++ b/doc/rsyslog_conf_examples.html @@ -0,0 +1,209 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Examples - rsyslog.conf</title></head> +<body> +<p>This is a part of the rsyslog.conf documentation.</p> +<a href="rsyslog_conf.html">back</a> +<h2>Examples</h2> +<p>Below are example for templates and selector lines. I hope +they are self-explanatory. If not, please see +www.monitorware.com/rsyslog/ for advise.</p> +<h3>TEMPLATES</h3> +<p>Please note that the samples are split across multiple lines. +A template MUST NOT actually be split across multiple lines.<br> +<br> +A template that resembles traditional syslogd file output:<br> +$template TraditionalFormat,"%timegenerated% %HOSTNAME%<br> +%syslogtag%%msg:::drop-last-lf%\n"<br> +<br> +A template that tells you a little more about the message:<br> +$template +precise,"%syslogpriority%,%syslogfacility%,%timegenerated%,%HOSTNAME%,<br> +%syslogtag%,%msg%\n"<br> +<br> +A template for RFC 3164 format:<br> +$template RFC3164fmt,"<%PRI%>%TIMESTAMP% %HOSTNAME% +%syslogtag%%msg%"<br> +<br> +A template for the format traditonally used for user messages:<br> +$template usermsg," XXXX%syslogtag%%msg%\n\r"<br> +<br> +And a template with the traditonal wall-message format:<br> +$template wallmsg,"\r\n\7Message from syslogd@%HOSTNAME% at +%timegenerated%<br> +<br> +A template that can be used for the database write (please note the SQL<br> +template option)<br> +$template MySQLInsert,"insert iut, message, receivedat values<br> +('%iut%', '%msg:::UPPERCASE%', '%timegenerated:::date-mysql%')<br> +into systemevents\r\n", SQL<br> +<br> +The following template emulates <a href="http://www.winsyslog.com/en/">WinSyslog</a> +format (it's an <a href="http://www.adiscon.com/en/">Adiscon</a> +format, you do not feel bad if you don't know it ;)). It's interesting +to see how it takes different parts out of the date stamps. What +happens is that the date stamp is split into the actual date and time +and the these two are combined with just a comma in between them.<br> +<br> +$template WinSyslogFmt,"%HOSTNAME%,%timegenerated:1:10:date-rfc3339%,<br> +%timegenerated:12:19:date-rfc3339%,%timegenerated:1:10:date-rfc3339%,<br> +%timegenerated:12:19:date-rfc3339%,%syslogfacility%,%syslogpriority%,<br> +%syslogtag%%msg%\n"</p> +<h3>SELECTOR LINES</h3> +<p># Store critical stuff in critical<br> +#<br> +*.=crit;kern.none /var/adm/critical<br> +<br> +This will store all messages with the priority crit in the file +/var/adm/critical, except for any kernel message.<br> +<br> +<br> +# Kernel messages are first, stored in the kernel<br> +# file, critical messages and higher ones also go<br> +# to another host and to the console. Messages to<br> +# the host finlandia are forwarded in RFC 3164<br> +# format (using the template defined above).<br> +#<br> +kern.* /var/adm/kernel<br> +kern.crit @finlandia;RFC3164fmt<br> +kern.crit /dev/console<br> +kern.info;kern.!err /var/adm/kernel-info<br> +<br> +The first rule direct any message that has the kernel facility to the +file /var/adm/kernel.<br> +<br> +The second statement directs all kernel messages of the priority crit +and higher to the remote host finlandia. This is useful, because if the +host crashes and the disks get irreparable errors you might not be able +to read the stored messages. If they're on a remote host, too, you +still can try to find out the reason for the crash.<br> +<br> +The third rule directs these messages to the actual console, so the +person who works on the machine will get them, too.<br> +<br> +The fourth line tells rsyslogd to save all kernel messages that come +with priorities from info up to warning in the file +/var/adm/kernel-info. Everything from err and higher is excluded.<br> +<br> +<br> +# The tcp wrapper loggs with mail.info, we display<br> +# all the connections on tty12<br> +#<br> +mail.=info /dev/tty12<br> +<br> +This directs all messages that uses mail.info (in source LOG_MAIL | +LOG_INFO) to /dev/tty12, the 12th console. For example the tcpwrapper +tcpd(8) uses this as it's default.<br> +<br> +<br> +# Store all mail concerning stuff in a file<br> +#<br> +mail.*;mail.!=info /var/adm/mail<br> +<br> +This pattern matches all messages that come with the mail facility, +except for the info priority. These will be stored in the file +/var/adm/mail.<br> +<br> +<br> +# Log all mail.info and news.info messages to info<br> +#<br> +mail,news.=info /var/adm/info<br> +<br> +This will extract all messages that come either with mail.info or with +news.info and store them in the file /var/adm/info.<br> +<br> +<br> +# Log info and notice messages to messages file<br> +#<br> +*.=info;*.=notice;\<br> +mail.none /var/log/messages<br> +<br> +This lets rsyslogd log all messages that come with either the info or +the notice facility into the file /var/log/messages, except for all<br> +messages that use the mail facility.<br> +<br> +<br> +# Log info messages to messages file<br> +#<br> +*.=info;\<br> +mail,news.none /var/log/messages<br> +<br> +This statement causes rsyslogd to log all messages that come with the +info priority to the file /var/log/messages. But any message coming +either with the mail or the news facility will not be stored.<br> +<br> +<br> +# Emergency messages will be displayed using wall<br> +#<br> +*.=emerg *<br> +<br> +This rule tells rsyslogd to write all emergency messages to all +currently logged in users. This is the wall action.<br> +<br> +<br> +# Messages of the priority alert will be directed<br> +# to the operator<br> +#<br> +*.alert root,rgerhards<br> +<br> +This rule directs all messages with a priority of alert or higher to +the terminals of the operator, i.e. of the users "root'' and +"rgerhards'' if they're logged in.<br> +<br> +<br> +*.* @finlandia<br> +<br> +This rule would redirect all messages to a remote host called +finlandia. This is useful especially in a cluster of machines where all +syslog messages will be stored on only one machine.<br> +<br> +In the format shown above, UDP is used for transmitting the message. +The destination port is set to the default auf 514. Rsyslog is also +capable of using much more secure and reliable TCP sessions for message +forwarding. Also, the destination port can be specified. To select TCP, +simply add one additional @ in front of the host name (that is, @host +is UPD, @@host is TCP). For example:<br> +<br> +<br> +*.* @@finlandia<br> +<br> +To specify the destination port on the remote machine, use a colon +followed by the port number after the machine name. The following +forwards to port 1514 on finlandia:<br> +<br> +<br> +*.* @@finlandia:1514<br> +<br> +This syntax works both with TCP and UDP based syslog. However, you will +probably primarily need it for TCP, as there is no well-accepted port +for this transport (it is non-standard). For UDP, you can usually stick +with the default auf 514, but might want to modify it for security rea-<br> +sons. If you would like to do that, it's quite easy:<br> +<br> +<br> +*.* @finlandia:1514<br> +<br> +<br> +<br> +*.* >dbhost,dbname,dbuser,dbpassword;dbtemplate<br> +<br> +This rule writes all message to the database "dbname" hosted on +"dbhost". The login is done with user "dbuser" and password +"dbpassword". The actual table that is updated is specified within the +template (which contains the insert statement). The template is called +"dbtemplate" in this case.</p> +<p>:msg,contains,"error" @errorServer</p> +<p>This rule forwards all messages that contain the word "error" +in the msg part to the server "errorServer". Forwarding is via UDP. +Please note the colon in fron</p> + +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> + diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html new file mode 100644 index 00000000..1d30d8ae --- /dev/null +++ b/doc/rsyslog_conf_filter.html @@ -0,0 +1,284 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Filter Conditions - rsyslog.conf</title></head> +<body> +<p>This is a part of the rsyslog.conf documentation.</p> +<a href="rsyslog_conf.html">back</a> +<h2>Filter Conditions</h2> +<p>Rsyslog offers four different types "filter conditions":</p> +<ul> +<li>BSD-style blocks</li> +<li>"traditional" severity and facility based selectors</li> +<li>property-based filters</li> +<li>expression-based filters</li> +</ul> +<h3>Blocks</h3> +<p>Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each +block of lines is separated from the previous block by a program or +hostname specification. A block will only log messages corresponding to +the most recent program and hostname specifications given. Thus, a +block which selects ‘ppp’ as the program, directly followed by a block +that selects messages from the hostname ‘dialhost’, then the second +block will only log messages from the ppp program on dialhost. +</p> +<p>A program specification is a line beginning with ‘!prog’ and +the following blocks will be associated with calls to syslog from that +specific program. A program specification for ‘foo’ will also match any +message logged by the kernel with the prefix ‘foo: ’. Alternatively, a +program specification ‘-foo’ causes the following blocks to be applied +to messages from any program but the one specified. A hostname +specification of the form ‘+hostname’ and the following blocks will be +applied to messages received from the specified hostname. +Alternatively, a hostname specification ‘-hostname’ causes the +following blocks to be applied to messages from any host but the one +specified. If the hostname is given as ‘@’, the local hostname will be +used. (NOT YET IMPLEMENTED) A program or hostname specification may be +reset by giving the program or hostname as ‘*’.</p> +<p>Please note that the "#!prog", "#+hostname" and "#-hostname" +syntax available in BSD syslogd is not supported by rsyslogd. By +default, no hostname or program is set.</p> +<h3>Selectors</h3> +<p><b>Selectors are the traditional way of filtering syslog +messages.</b> They have been kept in rsyslog with their original +syntax, because it is well-known, highly effective and also needed for +compatibility with stock syslogd configuration files. If you just need +to filter based on priority and facility, you should do this with +selector lines. They are <b>not</b> second-class citizens +in rsyslog and offer the best performance for this job.</p> +<p>The selector field itself again consists of two parts, a +facility and a priority, separated by a period (".''). Both parts are +case insensitive and can also be specified as decimal numbers, but +don't do that, you have been warned. Both facilities and priorities are +described in rsyslog(3). The names mentioned below correspond to the +similar LOG_-values in /usr/include/rsyslog.h.<br> +<br> +The facility is one of the following keywords: auth, authpriv, cron, +daemon, kern, lpr, mail, mark, news, security (same as auth), syslog, +user, uucp and local0 through local7. The keyword security should not +be used anymore and mark is only for internal use and therefore should +not be used in applications. Anyway, you may want to specify and +redirect these messages here. The facility specifies the subsystem that +produced the message, i.e. all mail programs log with the mail facility +(LOG_MAIL) if they log using syslog.<br> +<br> +The priority is one of the following keywords, in ascending order: +debug, info, notice, warning, warn (same as warning), err, error (same +as err), crit, alert, emerg, panic (same as emerg). The keywords error, +warn and panic are deprecated and should not be used anymore. The +priority defines the severity of the message.<br> +<br> +The behavior of the original BSD syslogd is that all messages of the +specified priority and higher are logged according to the given action. +Rsyslogd behaves the same, but has some extensions.<br> +<br> +In addition to the above mentioned names the rsyslogd(8) understands +the following extensions: An asterisk ("*'') stands for all facilities +or all priorities, depending on where it is used (before or after the +period). The keyword none stands for no priority of the given facility.<br> +<br> +You can specify multiple facilities with the same priority pattern in +one statement using the comma (",'') operator. You may specify as much +facilities as you want. Remember that only the facility part from such +a statement is taken, a priority part would be skipped.</p> +<p>Multiple selectors may be specified for a single action using +the semicolon (";'') separator. Remember that each selector in the +selector field is capable to overwrite the preceding ones. Using this +behavior you can exclude some priorities from the pattern.</p> +<p>Rsyslogd has a syntax extension to the original BSD source, +that makes its use more intuitively. You may precede every priority +with an equation sign ("='') to specify only this single priority and +not any of the above. You may also (both is valid, too) precede the +priority with an exclamation mark ("!'') to ignore all that +priorities, either exact this one or this and any higher priority. If +you use both extensions than the exclamation mark must occur before the +equation sign, just use it intuitively.</p> +<h3>Property-Based Filters</h3> +<p>Property-based filters are unique to rsyslogd. They allow to +filter on any property, like HOSTNAME, syslogtag and msg. A list of all +currently-supported properties can be found in the <a href="property_replacer.html">property replacer documentation</a> +(but keep in mind that only the properties, not the replacer is +supported). With this filter, each properties can be checked against a +specified value, using a specified compare operation.</p> +<p>A property-based filter must start with a colon in column 0. +This tells rsyslogd that it is the new filter type. The colon must be +followed by the property name, a comma, the name of the compare +operation to carry out, another comma and then the value to compare +against. This value must be quoted. There can be spaces and tabs +between the commas. Property names and compare operations are +case-sensitive, so "msg" works, while "MSG" is an invalid property +name. In brief, the syntax is as follows:</p> +<p><code><b>:property, [!]compare-operation, "value"</b></code></p> +<p>The following <b>compare-operations</b> are +currently supported:</p> +<table id="table1" border="1" width="100%"> +<tbody> +<tr> +<td>contains</td> +<td>Checks if the string provided in value is contained in +the property. There must be an exact match, wildcards are not supported.</td> +</tr> +<tr> +<td>isequal</td> +<td>Compares the "value" string provided and the property +contents. These two values must be exactly equal to match. The +difference to contains is that contains searches for the value anywhere +inside the property value, whereas all characters must be identical for +isequal. As such, isequal is most useful for fields like syslogtag or +FROMHOST, where you probably know the exact contents.</td> +</tr> +<tr> +<td>startswith</td> +<td>Checks if the value is found exactly at the beginning +of the property value. For example, if you search for "val" with +<p><code><b>:msg, startswith, "val"</b></code></p> +<p>it will be a match if msg contains "values are in this +message" but it won't match if the msg contains "There are values in +this message" (in the later case, contains would match). Please note +that "startswith" is by far faster than regular expressions. So even +once they are implemented, it can make very much sense +(performance-wise) to use "startswith".</p> +</td> +</tr> +<tr> +<td>regex</td> +<td>Compares the property against the provided POSIX +BRE regular +expression.</td> +</tr> +<tr> +<td>ereregex</td> +<td>Compares the property against the provided POSIX +ERE regular +expression.</td> +</tr> +</tbody> +</table> +<p>You can use the bang-character (!) immediately in front of a +compare-operation, the outcome of this operation is negated. For +example, if msg contains "This is an informative message", the +following sample would not match:</p> +<p><code><b>:msg, contains, "error"</b></code></p> +<p>but this one matches:</p> +<p><code><b>:msg, !contains, "error"</b></code></p> +<p>Using negation can be useful if you would like to do some +generic processing but exclude some specific events. You can use the +discard action in conjunction with that. A sample would be:</p> +<p><code><b>*.* +/var/log/allmsgs-including-informational.log<br> +:msg, contains, "informational" <font color="#ff0000" size="4">~</font> +<br> +*.* /var/log/allmsgs-but-informational.log</b></code></p> +<p>Do not overlook the red tilde in line 2! In this sample, all +messages are written to the file allmsgs-including-informational.log. +Then, all messages containing the string "informational" are discarded. +That means the config file lines below the "discard line" (number 2 in +our sample) will not be applied to this message. Then, all remaining +lines will also be written to the file allmsgs-but-informational.log.</p> +<p><b>Value</b> is a quoted string. It supports some +escape sequences:</p> +<p>\" - the quote character (e.g. "String with \"Quotes\"")<br> +\\ - the backslash character (e.g. "C:\\tmp")</p> +<p>Escape sequences always start with a backslash. Additional +escape sequences might be added in the future. Backslash characters <b>must</b> +be escaped. Any other sequence then those outlined above is invalid and +may lead to unpredictable results.</p> +<p>Probably, "msg" is the most prominent use case of property +based filters. It is the actual message text. If you would like to +filter based on some message content (e.g. the presence of a specific +code), this can be done easily by:</p> +<p><code><b>:msg, contains, "ID-4711"</b></code></p> +<p>This filter will match when the message contains the string +"ID-4711". Please note that the comparison is case-sensitive, so it +would not match if "id-4711" would be contained in the message.</p> +<p><code><b>:msg, regex, "fatal .* error"</b></code></p> +<p>This filter uses a POSIX regular expression. It matches when +the +string contains the words "fatal" and "error" with anything in between +(e.g. "fatal net error" and "fatal lib error" but not "fatal error" as +two spaces are required by the regular expression!).</p> +<p>Getting property-based filters right can sometimes be +challenging. In order to help you do it with as minimal effort as +possible, rsyslogd spits out debug information for all property-based +filters during their evaluation. To enable this, run rsyslogd in +foreground and specify the "-d" option.</p> +<p>Boolean operations inside property based filters (like +'message contains "ID17" or message contains "ID18"') are currently not +supported (except for "not" as outlined above). Please note that while +it is possible to query facility and severity via property-based +filters, it is far more advisable to use classic selectors (see above) +for those cases.</p> +<h3>Expression-Based Filters</h3> +Expression based filters allow +filtering on arbitrary complex expressions, which can include boolean, +arithmetic and string operations. Expression filters will evolve into a +full configuration scripting language. Unfortunately, their syntax will +slightly change during that process. So if you use them now, you need +to be prepared to change your configuration files some time later. +However, we try to implement the scripting facility as soon as possible +(also in respect to stage work needed). So the window of exposure is +probably not too long.<br> +<br> +Expression based filters are indicated by the keyword "if" in column 1 +of a new line. They have this format:<br> +<br> +if expr then action-part-of-selector-line<br> +<br> +"If" and "then" are fixed keywords that mus be present. "expr" is a +(potentially quite complex) expression. So the <a href="expression.html">expression documentation</a> for +details. "action-part-of-selector-line" is an action, just as you know +it (e.g. "/var/log/logfile" to write to that file).<br> +<br> +A few quick samples:<br> +<br> +<code> +*.* /var/log/file1 # the traditional way<br> +if $msg contains 'error' /var/log/errlog # the expression-based way<br> +</code> +<br> +Right now, you need to specify numerical values if you would like to +check for facilities and severity. These can be found in <a href="http://www.ietf.org/rfc/rfc3164.txt">RFC 3164</a>. +If you don't like that, you can of course also use the textual property +- just be sure to use the right one. As expression support is enhanced, +this will change. For example, if you would like to filter on message +that have facility local0, start with "DEVNAME" and have either +"error1" or "error0" in their message content, you could use the +following filter:<br> +<br> +<code> +if $syslogfacility-text == 'local0' and $msg +startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains +'error0') then /var/log/somelog<br> +</code> +<br> +Please note that the above <span style="font-weight: bold;">must +all be on one line</span>! And if you would like to store all +messages except those that contain "error1" or "error0", you just need +to add a "not":<br> +<br> +<code> +if $syslogfacility-text == 'local0' and $msg +startswith 'DEVNAME' and <span style="font-weight: bold;">not</span> +($msg contains 'error1' or $msg contains +'error0') then /var/log/somelog<br> +</code> +<br> +If you would like to do case-insensitive comparisons, use +"contains_i" instead of "contains" and "startswith_i" instead of +"startswith".<br> +<br> +Note that regular expressions are currently NOT +supported in expression-based filters. These will be added later when +function support is added to the expression engine (the reason is that +regular expressions will be a separate loadable module, which requires +some more prequisites before it can be implemented).<br> + +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> + diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html new file mode 100644 index 00000000..2371c4ae --- /dev/null +++ b/doc/rsyslog_conf_global.html @@ -0,0 +1,274 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Configuration Directives - rsyslog.conf</title></head> +<body> +<p>This is a part of the rsyslog.conf documentation.</p> +<a href="rsyslog_conf.html">back</a> +<h2>Configuration Directives</h2> +<p>All configuration directives need to be specified on a line by their +own and must start with a dollar-sign. Note that those starting with +the word "Action" modify the next action and should be specified +in front of it. +<p>Here is a list in alphabetical order. Follow links for a description.</p> +<p>Not all directives have an in-depth description right now. +Default values for them are in bold. A more in-depth description will +appear as implementation progresses. +</p> +<p><b>Be sure to read information about <a href="queues.html">queues in rsyslog</a></b> - +many parameter settings modify queue parameters. If in doubt, use the +default, it is usually well-chosen and applicable in most cases.</p> +<ul> +<li><a href="rsconf1_actionexeconlywhenpreviousissuspended.html">$ActionExecOnlyWhenPreviousIsSuspended</a></li> +<li>$ActionName <a_single_word> - used primarily for documentation, e.g. when +generating a configuration graph. Available sice 4.3.1. +<li>$ActionExecOnlyOnceEveryInterval <seconds> - +execute action only if the last execute is at last +<seconds> seconds in the past (more info in <a href="ommail.html">ommail</a>, +but may be used with any action)</li> +<li><i><b>$ActionExecOnlyEveryNthTime</b> <number></i> - If configured, the next action will +only be executed every n-th time. For example, if configured to 3, the first two messages +that go into the action will be dropped, the 3rd will actually cause the action to execute, +the 4th and 5th will be dropped, the 6th executed under the action, ... and so on. Note: +this setting is automatically re-set when the actual action is defined.</li> +<li><i><b>$ActionExecOnlyEveryNthTimeTimeout</b> <number-of-seconds></i> - has a meaning only if +$ActionExecOnlyEveryNthTime is also configured for the same action. If so, the timeout +setting specifies after which period the counting of "previous actions" expires and +a new action count is begun. Specify 0 (the default) to disable timeouts. +<br> +<i>Why is this option needed?</i> Consider this case: a message comes in at, eg., 10am. That's +count 1. Then, nothing happens for the next 10 hours. At 8pm, the next +one occurs. That's count 2. Another 5 hours later, the next message +occurs, bringing the total count to 3. Thus, this message now triggers +the rule. +<br> +The question is if this is desired behavior? Or should the rule only be +triggered if the messages occur within an e.g. 20 minute window? If the +later is the case, you need a +<br> +$ActionExecOnlyEveryNthTimeTimeout 1200 +<br> +This directive will timeout previous messages seen if they are older +than 20 minutes. In the example above, the count would now be always 1 +and consequently no rule would ever be triggered. + +<li>$ActionFileDefaultTemplate [templateName] - sets a new default template for file actions</li> +<li>$ActionFileEnableSync [on/<span style="font-weight: bold;">off</span>] - enables file +syncing capability of omfile</li> +<li>$ActionForwardDefaultTemplate [templateName] - sets a new +default template for UDP and plain TCP forwarding action</li> +<li>$ActionGSSForwardDefaultTemplate [templateName] - sets a +new default template for GSS-API forwarding action</li> +<li>$ActionQueueCheckpointInterval <number></li> +<li>$ActionQueueDequeueBatchSize <number> [default 16]</li> +<li>$ActionQueueDequeueSlowdown <number> [number +is timeout in <i> micro</i>seconds (1000000us is 1sec!), +default 0 (no delay). Simple rate-limiting!]</li> +<li>$ActionQueueDiscardMark <number> [default +9750]</li> +<li>$ActionQueueDiscardSeverity <number> +[*numerical* severity! default 4 (warning)]</li> +<li>$ActionQueueFileName <name></li> +<li>$ActionQueueHighWaterMark <number> [default +8000]</li> +<li>$ActionQueueImmediateShutdown [on/<b>off</b>]</li> +<li>$ActionQueueSize <number></li> +<li>$ActionQueueLowWaterMark <number> [default +2000]</li> +<li>$ActionQueueMaxFileSize <size_nbr>, default 1m</li> +<li>$ActionQueueTimeoutActionCompletion <number> +[number is timeout in ms (1000ms is 1sec!), default 1000, 0 means +immediate!]</li> +<li>$ActionQueueTimeoutEnqueue <number> [number +is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li> +<li>$ActionQueueTimeoutShutdown <number> [number +is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li> +<li>$ActionQueueWorkerTimeoutThreadShutdown +<number> [number is timeout in ms (1000ms is 1sec!), +default 60000 (1 minute)]</li> +<li>$ActionQueueType [FixedArray/LinkedList/<b>Direct</b>/Disk]</li> +<li>$ActionQueueSaveOnShutdown [on/<b>off</b>] +</li> +<li>$ActionQueueWorkerThreads <number>, num worker threads, default 1, recommended 1</li> +<li>$ActionQueueWorkerThreadMinumumMessages <number>, default 100</li> +<li><a href="rsconf1_actionresumeinterval.html">$ActionResumeInterval</a></li> +<li>$ActionResumeRetryCount <number> [default 0, -1 means eternal]</li> +<li>$ActionSendResendLastMsgOnReconn <[on/<b>off</b>]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. +<li>$ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action +</li><li>$ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver +(driver-specific)</li><li>$ActionSendStreamDriverAuthMode <mode>, authentication mode to use with the stream driver +(driver-specific)</li><li>$ActionSendStreamDriverPermittedPeer <ID>, accepted fingerprint (SHA1) or name of remote peer +(driver-specific) -<span style="font-weight: bold;"> directive may go away</span>!</li> +<li><b>$ActionSendUDPRebindInterval</b> nbr</a>- [available since 4.3.2] - instructs the UDP send +action to rebind the send socket every nbr of messages sent. Zero, the default, means +that no rebind is done. This directive is useful for use with load-balancers.</li> +<li><a href="rsconf1_allowedsender.html">$AllowedSender</a></li> +<li><a href="rsconf1_controlcharacterescapeprefix.html">$ControlCharacterEscapePrefix</a></li> +<li><a href="rsconf1_debugprintcfsyslinehandlerlist.html">$DebugPrintCFSyslineHandlerList</a></li> + +<li><a href="rsconf1_debugprintmodulelist.html">$DebugPrintModuleList</a></li> +<li><a href="rsconf1_debugprinttemplatelist.html">$DebugPrintTemplateList</a></li> +<li>$DefaultNetstreamDriver <drivername>, the default <a href="netstream.html">network stream driver</a> to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem></li> +<li>$DefaultNetstreamDriverCertFile </path/to/certfile.pem></li> +<li>$DefaultNetstreamDriverKeyFile </path/to/keyfile.pem></li> +<li><b>$DefaultRuleset</b> <i>name</i> - changes the default ruleset for unbound inputs to +the provided <i>name</i> (the default default ruleset is named +"RSYSLOG_DefaultRuleset"). It is advised to also read +our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a>.</li> +<li><b>$CreateDirs</b> [<b>on</b>/off] - create directories on an as-needed basis</li> +<li><a href="rsconf1_dircreatemode.html">$DirCreateMode</a></li> +<li><a href="rsconf1_dirgroup.html">$DirGroup</a></li> +<li><a href="rsconf1_dirowner.html">$DirOwner</a></li> +<li><a href="rsconf1_dropmsgswithmaliciousdnsptrrecords.html">$DropMsgsWithMaliciousDnsPTRRecords</a></li> +<li><a href="rsconf1_droptrailinglfonreception.html">$DropTrailingLFOnReception</a></li> +<li><a href="rsconf1_dynafilecachesize.html">$DynaFileCacheSize</a></li> +<li><a href="rsconf1_escapecontrolcharactersonreceive.html">$EscapeControlCharactersOnReceive</a></li> +<li>$ErrorMessagesToStderr [<b>on</b>|off] - direct rsyslogd error message to stderr (in addition to other targets)</li> +<li><a href="rsconf1_failonchownfailure.html">$FailOnChownFailure</a></li> +<li><a href="rsconf1_filecreatemode.html">$FileCreateMode</a></li> +<li><a href="rsconf1_filegroup.html">$FileGroup</a></li> +<li><a href="rsconf1_fileowner.html">$FileOwner</a></li> +<li><a href="rsconf1_generateconfiggraph.html">$GenerateConfigGraph</a></li> +<li><a href="rsconf1_gssforwardservicename.html">$GssForwardServiceName</a></li> +<li><a href="rsconf1_gsslistenservicename.html">$GssListenServiceName</a></li> +<li><a href="rsconf1_gssmode.html">$GssMode</a></li> +<li>$HUPisRestart [<b>on</b>/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending +on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is +not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for +compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually +the only one that is needed e.g. for log rotation. <b>It is recommended to set the setting to "off".</b></li> +<li><a href="rsconf1_includeconfig.html">$IncludeConfig</a></li><li>MainMsgQueueCheckpointInterval <number></li> +<li>$MainMsgQueueDequeueBatchSize <number> [default 32]</li> +<li>$MainMsgQueueDequeueSlowdown <number> [number +is timeout in <i> micro</i>seconds (1000000us is 1sec!), +default 0 (no delay). Simple rate-limiting!]</li> +<li>$MainMsgQueueDiscardMark <number> [default 9750]</li> +<li>$MainMsgQueueDiscardSeverity <severity> +[either a textual or numerical severity! default 4 (warning)]</li> +<li>$MainMsgQueueFileName <name></li> +<li>$MainMsgQueueHighWaterMark <number> [default +8000]</li> +<li>$MainMsgQueueImmediateShutdown [on/<b>off</b>]</li> +<li><a href="rsconf1_mainmsgqueuesize.html">$MainMsgQueueSize</a></li> +<li>$MainMsgQueueLowWaterMark <number> [default +2000]</li> +<li>$MainMsgQueueMaxFileSize <size_nbr>, default +1m</li> +<li>$MainMsgQueueTimeoutActionCompletion +<number> [number is timeout in ms (1000ms is 1sec!), +default +1000, 0 means immediate!]</li> +<li>$MainMsgQueueTimeoutEnqueue <number> [number +is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]</li> +<li>$MainMsgQueueTimeoutShutdown <number> [number +is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]</li> +<li>$MainMsgQueueWorkerTimeoutThreadShutdown +<number> [number is timeout in ms (1000ms is 1sec!), +default 60000 (1 minute)]</li> +<li>$MainMsgQueueType [<b>FixedArray</b>/LinkedList/Direct/Disk]</li> +<li>$MainMsgQueueSaveOnShutdown [on/<b>off</b>] +</li> +<li>$MainMsgQueueWorkerThreads <number>, num +worker threads, default 1, recommended 1</li> +<li>$MainMsgQueueWorkerThreadMinumumMessages <number>, default 100</li> +<li><a href="rsconf1_markmessageperiod.html">$MarkMessagePeriod</a> (immark)</li> +<li><b><i>$MaxMessageSize</i></b> <size_nbr>, default 2k - allows to specify maximum supported message size +(both for sending and receiving). The default +should be sufficient for almost all cases. Do not set this below 1k, as it would cause +interoperability problems with other syslog implementations.<br> +Change the setting to e.g. 32768 if you would like to +support large message sizes for IHE (32k is the current maximum +needed for IHE). I was initially tempted to set the default to 32k, +but there is a some memory footprint with the current +implementation in rsyslog. +<br>If you intend to receive Windows Event Log data (e.g. via +<a href="http://www.eventreporter.com/">EventReporter</a>), you might want to +increase this number to an even higher value, as event +log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea). +Note: testing showed that 4k seems to be +the typical maximum for <b>UDP</b> based syslog. This is an IP stack +restriction. Not always ... but very often. If you go beyond +that value, be sure to test that rsyslogd actually does what +you think it should do ;) It is highly suggested to use a TCP based transport +instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions. +<br>Note that 2k, the current default, is the smallest size that must be +supported in order to be compliant to the upcoming new syslog RFC series. +</li> +<li><a href="rsconf1_maxopenfiles.html">$MaxOpenFiles</a></li> +<li><a href="rsconf1_moddir.html">$ModDir</a></li> +<li><a href="rsconf1_modload.html">$ModLoad</a></li> +<li><b>$OMFileZipLevel</b> 0..9 [default 0] - if greater 0, turns on gzip compression +of the output file. The higher the number, the better the compression, but also the +more CPU is required for zipping.</li> +<li><b>$OMFileIOBufferSize</b> <size_nbr>, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)</li> +<li><b>$OMFileFlushOnTXEnd</b> <[<b>on</b>/off]>, default on. Omfile has the +capability to +writes output using a buffered writer. Disk writes are only done when the buffer is +full. So if an error happens during that write, data is potentially lost. In cases where +this is unacceptable, set $OMFileFlushOnTXEnd to on. Then, data is written at the end +of each transaction (for pre-v5 this means after <b>each</b> log message) and the usual +error recovery thus can handle write errors without data loss. Note that this option +severely reduces the effect of zip compression and should be switched to off +for that use case. Note that the default -off- is primarily an aid to preserve +the traditional syslogd behaviour.</li> +<li><b>$RepeatedMsgContainsOriginalMsg</b> [on/<b>off</b>] - "last message repeated n times" messages, if generated, +have a different format that contains the message that is being repeated. +Note that only the first "n" characters are included, with n to be at least 80 characters, most +probably more (this may change from version to version, thus no specific limit is given). The bottom +line is that n is large enough to get a good idea which message was repeated but it is not necessarily +large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.</li> +<li><a href="rsconf1_repeatedmsgreduction.html">$RepeatedMsgReduction</a></li> +<li><a href="rsconf1_resetconfigvariables.html">$ResetConfigVariables</a></li> +<li><b>$Ruleset</b> <i>name</i> - starts a new ruleset or switches back to one already defined. +All following actions belong to that new rule set. +the <i>name</i> does not yet exist, it is created. To swith back to rsyslog's +default ruleset, specify "RSYSLOG_DefaultRuleset") as the name. +All following actions belong to that new rule set. It is advised to also read +our paper on <a href="multi_ruleset.html">using multiple rule sets in rsyslog</a>.</li> +<li><b>$OptimizeForUniprocessor</b> [on/<b>off</b>] - turns on optimizatons which lead to better +performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The +default may change as uniprocessor systems become less common. [available since 4.1.0]</li> +<li>$PreserveFQDN [on/<b>off</b>) - if set to off (legacy default to remain compatible +to sysklogd), the domain part from a name that is within the same domain as the receiving +system is stripped. If set to on, full names are always used.</li> +<li>$WorkDirectory <name> (directory for spool and other work files)</li> +<li>$UDPServerAddress <IP> (imudp) -- local IP +address (or name) the UDP listens should bind to</li> +<li>$UDPServerRun <port> (imudp) -- former +-r<port> option, default 514, start UDP server on this +port, "*" means all addresses</li> +<li>$UDPServerTimeRequery <nbr-of-times> (imudp) -- this is a performance +optimization. Getting the system time is very costly. With this setting, imudp can +be instructed to obtain the precise time only once every n-times. This logic is +only activated if messages come in at a very fast rate, so doing less frequent +time calls should usually be acceptable. The default value is two, because we have +seen that even without optimization the kernel often returns twice the identical time. +You can set this value as high as you like, but do so at your own risk. The higher +the value, the less precise the timestamp. +<li><a href="droppriv.html">$PrivDropToGroup</a></li> +<li><a href="droppriv.html">$PrivDropToGroupID</a></li> +<li><a href="droppriv.html">$PrivDropToUser</a></li> +<li><a href="droppriv.html">$PrivDropToUserID</a></li> +<li><a href="rsconf1_umask.html">$UMASK</a></li> +</ul> +<p><b>Where <size_nbr> is specified above,</b> +modifiers can be used after the number part. For example, 1k means +1024. Supported are k(ilo), m(ega), g(iga), t(era), p(eta) and e(xa). +Lower case letters refer to the traditional binary defintion (e.g. 1m +equals 1,048,576) whereas upper case letters refer to their new +1000-based definition (e.g 1M equals 1,000,000).</p> +<p>Numbers may include '.' and ',' for readability. So you can +for example specify either "1000" or "1,000" with the same result. +Please note that rsyslogd simply ignores the punctuation. Form it's +point of view, "1,,0.0.,.,0" also has the value 1000. </p> + +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> + + diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html new file mode 100644 index 00000000..df9abeea --- /dev/null +++ b/doc/rsyslog_conf_modules.html @@ -0,0 +1,53 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Modules - rsyslog.conf</title></head> +<body> +<p>This is a part of the rsyslog.conf documentation.</p> +<a href="rsyslog_conf.html">back</a> +<h2>Modules</h2> +<p>Rsyslog has a modular design. Consequently, there is a growing +number of modules. Here is the entry point to their documentation and +what they do (list is currently not complete)</p> +<ul> +<li><a href="omsnmp.html">omsnmp</a> - SNMP trap output module</li> +<li><a href="omstdout.html">omtdout</a> - stdout output module (mainly a test tool)</li> +<li><a href="omrelp.html">omrelp</a> - RELP output module</li> +<li>omgssapi - output module for GSS-enabled syslog</li> +<li><a href="ommysql.html">ommysql</a> - output module for MySQL</li> +<li>ompgsql - output module for PostgreSQL</li> +<li><a href="omlibdbi.html">omlibdbi</a> - +generic database output module (Firebird/Interbase, MS SQL, Sybase, +SQLLite, Ingres, Oracle, mSQL)</li> +<li><a href="ommail.html">ommail</a> - +permits rsyslog to alert folks by mail if something important happens</li> +<li><a href="omoracle.html">omoracle</a> - output module for Oracle (native OCI interface)</li> +<li><a href="imfile.html">imfile</a> +- input module for text files</li> +<li><a href="imrelp.html">imrelp</a> - RELP +input module</li> +<li>imudp - udp syslog message input</li> +<li><a href="imtcp.html">imtcp</a> - input +plugin for plain tcp syslog</li> +<li><a href="imgssapi.html">imgssapi</a> - +input plugin for plain tcp and GSS-enabled syslog</li> +<li>immark - support for mark messages</li> +<li><a href="imklog.html">imklog</a> - kernel logging</li> +<li><a href="imuxsock.html">imuxsock</a> - +unix sockets, including the system log socket</li> +<li><a href="im3195.html">im3195</a> - +accepts syslog messages via RFC 3195</li> +</ul> +<p>Please note that each module provides configuration +directives, which are NOT necessarily being listed below. Also +remember, that a modules configuration directive (and functionality) is +only available if it has been loaded (using $ModLoad).</p> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008, 2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 3 or higher.</font></p> +</body> +</html> + diff --git a/doc/rsyslog_conf_nomatch.html b/doc/rsyslog_conf_nomatch.html new file mode 100644 index 00000000..5f25f3e4 --- /dev/null +++ b/doc/rsyslog_conf_nomatch.html @@ -0,0 +1,48 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>nomatch mode - property replacer - rsyslog.conf</title></head> +<body> +<h1>nomatch mode - property replacer - rsyslog.con</h1> +<p>This is a part of the <a href="rsyslog_conf.html">rsyslog.conf documentation</a> +of the <a href="property_replacer.html">property replacer</a>.</p> +<p><b>The "nomatch-Mode" specifies which string the property replacer +shall return if a regular expression did not find the search string.</b>. Traditionally, +the string "**NO MATCH**" was returned, but many people complained this was almost never useful. +Still, this mode is support as "<b>DFLT</b>" for legacy configurations. +<p>Three additional and potentially useful modes exist: in one (<b>BLANK</b>) a blank string +is returned. This is probably useful for inserting values into databases where no +value shall be inserted if the expression could not be found. +<p>A similar mode is "<b>ZERO</b>" where the string "0" is returned. This is suitable +for numerical values. A use case may be +that you record a traffic log based on firewall rules and the "bytes transmitted" counter +is extracted via a regular expression. If no "bytes transmitted" counter is available +in the current message, it is probably a good idea to return an empty string, which the +database layer can turn into a zero. +<p>The other mode is "<b>FIELD</b>", in which the complete field is returned. This may be useful +in cases where absense of a match is considered a failure and the message that triggered +it shall be logged. +<p>If in doubt, <b>it is highly suggested to use the +<a href="http://www.rsyslog.com/tool-regex">rsyslog online regular expression +checker and generator</a> to see these options in action</b>. With that online tool, +you can craft regular expressions based on samples and try out the different modes. + +<h2>Summary of nomatch Modes</h2> +<table border="1" cellspacing="0"> +<tr><td><b>Mode</b></td><td><b>Returned</b></td></tr> +<tr><td>DFLT</td><td>"**NO MATCH**"</td></tr> +<tr><td>BLANK</td><td>"" (empty string)</td></tr> +<tr><td>ZERO</td><td>"0"</td></tr> +<tr><td>FIELD</td><td>full content of original field</td></tr> +<tr><td> </td><td><a href="http://www.rsyslog.com/tool-regex">Interactive Tool</a></td></tr> +</table> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> + + diff --git a/doc/rsyslog_conf_output.html b/doc/rsyslog_conf_output.html new file mode 100644 index 00000000..c52aaa5e --- /dev/null +++ b/doc/rsyslog_conf_output.html @@ -0,0 +1,81 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Output Channels - rsyslog.conf</title></head> +<body> +<p>This is a part of the rsyslog.conf documentation.</p> +<a href="rsyslog_conf.html">back</a> +<h2>Output Channels</h2> +<p>Output Channels are a new concept first introduced in rsyslog +0.9.0. <b>As of this writing, it is most likely that they will +be replaced by something different in the future.</b> So if you +use them, be prepared to change you configuration file syntax when you +upgrade to a later release.<br> +<br> +The idea behind output channel definitions is that it shall provide an +umbrella for any type of output that the user might want. In essence,<br> +this is the "file" part of selector lines (and this is why we are not +sure output channel syntax will stay after the next review). There is a<br> +difference, though: selector channels both have filter conditions +(currently facility and severity) as well as the output destination. +they can only be used to write to files - not pipes, ttys or whatever +Output channels define the output definition, only. As of this build, +else. If we stick with output channels, this will change over time.</p> +<p>In concept, an output channel includes everything needed to +know about an output actions. In practice, the current implementation +only carries<br> +a filename, a maximum file size and a command to be issued when this +file size is reached. More things might be present in future version, +which might also change the syntax of the directive.</p> +<p>Output channels are defined via an $outchannel directive. It's +syntax is as follows:<br> +<br> +$outchannel name,file-name,max-size,action-on-max-size<br> +<br> +name is the name of the output channel (not the file), file-name is the +file name to be written to, max-size the maximum allowed size and +action-on-max-size a command to be issued when the max size is reached. +This command always has exactly one parameter. The binary is that part +of action-on-max-size before the first space, its parameter is +everything behind that space.<br> +<br> +Please note that max-size is queried BEFORE writing the log message to +the file. So be sure to set this limit reasonably low so that any +message might fit. For the current release, setting it 1k lower than +you expected is helpful. The max-size must always be specified in bytes +- there are no special symbols (like 1k, 1m,...) at this point of +development.<br> +<br> +Keep in mind that $outchannel just defines a channel with "name". It +does not activate it. To do so, you must use a selector line (see +below). That selector line includes the channel name plus an $ sign in +front of it. A sample might be:<br> +<br> +*.* $mychannel<br> +<br> +In its current form, output channels primarily provide the ability to +size-limit an output file. To do so, specify a maximum size. When this +size is reached, rsyslogd will execute the action-on-max-size command +and then reopen the file and retry. The command should be something +like a <a href="log_rotation_fix_size.html">log rotation +script</a> or a similar thing.</p> +<p>If there is no action-on-max-size command or the command did +not resolve the situation, the file is closed and never reopened by +rsyslogd (except, of course, by huping it). This logic was integrated +when we first experienced severe issues with files larger 2gb, which +could lead to rsyslogd dumping core. In such cases, it is more +appropriate to stop writing to a single file. Meanwhile, rsyslogd has +been fixed to support files larger 2gb, but obviously only on file +systems and operating system versions that do so. So it can still make +sense to enforce a 2gb file size limit.</p> + +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> + + diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html new file mode 100644 index 00000000..6c68b801 --- /dev/null +++ b/doc/rsyslog_conf_templates.html @@ -0,0 +1,151 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Templates - rsyslog.conf</title></head> +<body> +<p>This is a part of the rsyslog.conf - documentation.</p> +<a href="rsyslog_conf.html">back</a> +<h2>Templates</h2> +<p>Templates are a key feature of rsyslog. They allow to specify +any +format a user might want. They are also used for dynamic file name +generation. Every output in rsyslog uses templates - this holds true +for files, user messages and so on. The database writer expects its +template to be a proper SQL statement - so this is highly customizable +too. You might ask how does all of this work when no templates at all +are specified. Good question ;) The answer is simple, though. Templates +compatible with the stock syslogd formats are hardcoded into rsyslogd. +So if no template is specified, we use one of these hardcoded +templates. Search for "template_" in syslogd.c and you will find the +hardcoded ones.</p> +<p>A template consists of a template directive, a name, the +actual template text and optional options. A sample is:</p> +<blockquote><code>$template MyTemplateName,"\7Text +%property% some more text\n",<options></code></blockquote> +<p>The "$template" is the template directive. It tells rsyslog +that this line contains a template. "MyTemplateName" is the template +name. All +other config lines refer to this name. The text within quotes is the +actual template text. The backslash is an escape character, much as it +is in C. It does all these "cool" things. For example, \7 rings the +bell (this is an ASCII value), \n is a new line. C programmers and perl +coders have the advantage of knowing this, but the set in rsyslog is a +bit restricted currently. +</p> +<p>All text in the template is used literally, except for things +within percent signs. These are properties and allow you access to the +contents of the syslog message. Properties are accessed via the +<a href="property_replacer.html">property replacer</a> +(nice name, huh) and it can do cool things, too. For +example, it can pick a substring or do date-specific formatting. More +on this is below, on some lines of the property replacer.<br> +<br> +The <options> part is optional. It carries options +influencing the template as whole. See details below. Be sure NOT to +mistake template options with property options - the later ones are +processed by the property replacer and apply to a SINGLE property, only +(and not the whole template).<br> +<br> +Template options are case-insensitive. Currently defined are: </p> +<p><b>sql</b> - format the string suitable for a SQL +statement in MySQL format. This will replace single quotes ("'") and +the backslash character by their backslash-escaped counterpart ("\'" +and "\\") inside each field. Please note that in MySQL configuration, +the <code class="literal">NO_BACKSLASH_ESCAPES</code> +mode must be turned off for this format to work (this is the default).</p> +<p><b>stdsql</b> - format the string suitable for a +SQL statement that is to be sent to a standards-compliant sql server. +This will replace single quotes ("'") by two single quotes ("''") +inside each field. You must use stdsql together with MySQL if in MySQL +configuration the +<code class="literal">NO_BACKSLASH_ESCAPES</code> is +turned on.</p> +<p>Either the <b>sql</b> or <b>stdsql</b> +option <b>must</b> be specified when a template is used +for writing to a database, otherwise injection might occur. Please note +that due to the unfortunate fact that several vendors have violated the +sql standard and introduced their own escape methods, it is impossible +to have a single option doing all the work. So you yourself +must make sure you are using the right format. <b>If you choose +the wrong one, you are still vulnerable to sql injection.</b><br> +<br> +Please note that the database writer *checks* that the sql option is +present in the template. If it is not present, the write database +action is disabled. This is to guard you against accidental forgetting +it and then becoming vulnerable to SQL injection. The sql option can +also be useful with files - especially if you want to import them into +a database on another machine for performance reasons. However, do NOT +use it if you do not have a real need for it - among others, it takes +some toll on the processing time. Not much, but on a really busy system +you might notice it ;)</p> +<p>The default template for the write to database action has the +sql option set. As we currently support only MySQL and the sql option +matches the default MySQL configuration, this is a good choice. +However, if you have turned on +<code class="literal">NO_BACKSLASH_ESCAPES</code> in +your MySQL config, you need to supply a template with the stdsql +option. Otherwise you will become vulnerable to SQL injection. <br> +<br> +To escape:<br> +% = \%<br> +\ = \\ --> '\' is used to escape (as in C)<br> +$template TraditionalFormat,%timegenerated% %HOSTNAME% +%syslogtag%%msg%\n"<br> +<br> +Properties can be accessed by the <a href="property_replacer.html">property +replacer</a> (see there for details).</p> +<p><b>Please note that templates can also by +used to generate selector lines with dynamic file names.</b> For +example, if you would like to split syslog messages from different +hosts to different files (one per host), you can define the following +template:</p> +<blockquote><code>$template +DynFile,"/var/log/system-%HOSTNAME%.log"</code></blockquote> +<p>This template can then be used when defining an output +selector line. It will result in something like +"/var/log/system-localhost.log"</p> +<p>Template +names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT +use them if, otherwise you may receive a conflict in the future (and +quite unpredictable behaviour). There is a small set of pre-defined +templates that you can use without the need to define it:</p> +<ul> +<li><span style="font-weight: bold;">RSYSLOG_TraditionalFileFormat</span> +- the "old style" default log file format with low-precision timestamps</li> +<li><span style="font-weight: bold;">RSYSLOG_FileFormat</span> +- a modern-style logfile format similar to TraditionalFileFormat, buth +with high-precision timestamps and timezone information</li> +<li><span style="font-weight: bold;">RSYSLOG_TraditionalForwardFormat</span> +- the traditional forwarding format with low-precision timestamps. Most +useful if you send messages to other syslogd's or rsyslogd +below +version 3.12.5.</li> +<li><span style="font-weight: bold;">RSYSLOG_ForwardFormat</span> +- a new high-precision forwarding format very similar to the +traditional one, but with high-precision timestamps and timezone +information. Recommended to be used when sending messages to rsyslog +3.12.5 or above.</li> +<li><span style="font-weight: bold;">RSYSLOG_SyslogProtocol23Format</span> +- the format specified in IETF's internet-draft +ietf-syslog-protocol-23, which is assumed to be come the new syslog +standard RFC. This format includes several improvements. The rsyslog +message parser understands this format, so you can use it together with +all relatively recent versions of rsyslog. Other syslogd's may get +hopelessly confused if receiving that format, so check before you use +it. Note that the format is unlikely to change when the final RFC comes +out, but this may happen.</li> +<li><span style="font-weight: bold;">RSYSLOG_DebugFormat</span> +- a special format used for troubleshooting property problems. This format +is meant to be written to a log file. Do <b>not</b> use for production or remote +forwarding.</li> +</ul> + +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body> +</html> + diff --git a/doc/rsyslog_confgraph_complex.conf b/doc/rsyslog_confgraph_complex.conf new file mode 100644 index 00000000..3d7ec0a3 --- /dev/null +++ b/doc/rsyslog_confgraph_complex.conf @@ -0,0 +1,108 @@ +$DebugPrintTemplateList off +$DebugPrintCfSysLineHandlerList off +$DebugPrintModuleList off +#$ResetConfigVariables +$ErrorMessagesToStderr off +$ModLoad /home/rger/proj/rsyslog/plugins/imuxsock/.libs/imuxsock.so +#$ModLoad /home/rger/proj/rsyslog/plugins/imklog/.libs/imklog +#$ModLoad /home/rger/proj/rsyslog/plugins/imtcp/.libs/imtcp +$ModLoad /home/rger/proj/rsyslog/plugins/imtcp/.libs/imtcp +$ModLoad /home/rger/proj/rsyslog/plugins/imudp/.libs/imudp +$ModLoad /home/rger/proj/rsyslog/plugins/omstdout/.libs/omstdout +$ModLoad /home/rger/proj/rsyslog/plugins/omprog/.libs/omprog +$ModLoad /home/rger/proj/rsyslog/plugins/omtesting/.libs/omtesting +#$ModLoad /home/rger/proj/rsyslog/plugins/ommail/.libs/ommail +# +# +# PGSQL testing +$ModLoad /home/rger/proj/rsyslog/plugins/ompgsql/.libs/ompgsql.so +$template pgfmt,"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%');",STDSQL +#$ActionQueueType linkedlist +#*.* :ompgsql:127.0.0.1,rsyslog,postgres,;pgfmt + +#$ActionOMStdoutArrayInterface on +#*.* :omstdout: + +$ActionResumeInterval 4 +$ActionResumeRetryCount 3 +$ActionQueueType LinkedList # run asynchronously +$ActionName Forward to 172.19.3.9 +*.* @@172.19.3.9:10514 +#*.* :omtesting:randfail +#*.* :omtesting:always_suspend +#*.* :omtesting:fail 2 2 + +#$UDPServerTimeRequery 10 +$UDPServerRun 514 +$inputtcpmaxsessions 2000 +$InputTCPServerRun 12514 + +#$PrivDropToUser rger +#$InputTCPServerInputName tcp/514 +#$InputTCPServerAddtlFrameDelimiter 10 +#$InputTCPServerRun 514 +#$AllowedSender UDP,127.0.0.1/32 +#$AllowedSender TCP,127.0.0.1/32 + +$PreserveFQDN off + +#$HUPisRestart on + +#$MainMsgQueueType direct +$MainMsgQueueType linkedlist +$MainMsgQueueDequeueBatchSize 200 +#$MainMsgQueueWorkerTimeoutThreadShutdown -1 + +#---- test DA mode +# set spool locations and switch queue to disk assisted mode +$WorkDirectory spool +$MainMsgQueueSize 200 # this *should* trigger moving on to DA mode... +# note: we must set QueueSize sufficiently high, so that 70% (light delay mark) +# is high enough above HighWatermark! +$MainMsgQueueHighWatermark 80 +$MainMsgQueueLowWatermark 40 +$MainMsgQueueFilename mainq +$MainMsgQueueType linkedlist +# ucomment, as we now have an issue (finally the test case works ;)) +#$MainMsgQueueDequeueBatchSize 80 +#---- end test DA mode + +#$template test,"%timereported:::date-rfc3339%,%timereported:::date-mysql%,%timereported:::date-subseconds%, %timegenerated:::date-mysql%, %timegenerated:::date-subseconds%, msg: %msg%\n" +#$template db,"re: '%msg:R,ERE,1,FIELD:dsn=([0-9]+\.[0-9]+\.[0-9])--end%', msg: '%msg%'\n" +#$template db,"re: '%msg:R,ERE,1,ZERO:dsn=([0-9]+\.[0-9]+\.[0-9])--end%', msg: '%msg%'\n" +#$template DEBUG,"Debug line with all properties:\nFROMHOST: '%FROMHOST%', fromhost-ip: '%fromhost-ip%, HOSTNAME: '%HOSTNAME%', PRI: %PRI%,\nsyslogtag '%syslogtag%', programname: '%programname%', APP-NAME: '%APP-NAME%', PROCID: '%PROCID%', MSGID: '%MSGID%',\nTIMESTAMP: '%TIMESTAMP%', STRUCTURED-DATA: '%STRUCTURED-DATA%',\nmsg: '%msg%'\nescaped msg: '%msg:::drop-cc%'\nrawmsg: '%rawmsg%'\n\n" +$template csv,"%syslogtag:::csv%,%msg:::upppercase,csv%,%msg%\n" +*.* -/home/rger/proj/rsyslog/logfile +kern.* -/home/rger/proj/rsyslog/logfile +$ActionExecOnlyWhenPreviousIsSuspended on +& -/tmp/xyz/uuu +$ActionExecOnlyWhenPreviousIsSuspended off +& ~ +& -/tmp/xyz/uuu2 +& -/tmp/xyz/uuu3 + + +#$template dynfile,"/home/rger/proj/rsyslog/test-%syslogtag%" +#*.* -?dynfile +#:msg, ereregex, "test|tast" /home/rger/proj/rsyslog/ere +#if strlen($syslogtag & strlen($msg)) > 10 then /home/rger/proj/rsyslog/longlog +#if strlen($msg) > 10 then /home/rger/proj/rsyslog/longlog +#if tolower($msg) contains 'test' then /home/rger/proj/rsyslog/longlog +#if $msg contains 'test' then /home/rger/proj/rsyslog/longlog + +#$ActionOMProgBinary /home/rger/proj/rsyslog/consumer +#*.* :omprog: + +#$actionresumeretryCount -1 +#$actionResumeInterval 4 +#$template dynfile,"/mnt2/logs/logfile.log" +#*.* /mnt2/logs/logfile.log +#if $msg contains 'test' then ?dynfile +#*.* ?dynfile +:msg, contains, "test " /tmpo/sdafsdf + +$ActionName write_system_log_2 +if $msg == 'test' then /tmpo/sdafsdf2 +& /tmpo/234234 +*.* @@(o,z9)172.19.3.21:10514 +$GenerateConfigGraph /home/rger/proj/rsyslog/rsyslog.dot diff --git a/doc/rsyslog_confgraph_complex.png b/doc/rsyslog_confgraph_complex.png Binary files differnew file mode 100644 index 00000000..21c04c57 --- /dev/null +++ b/doc/rsyslog_confgraph_complex.png diff --git a/doc/rsyslog_confgraph_std.conf b/doc/rsyslog_confgraph_std.conf new file mode 100644 index 00000000..64c9a18a --- /dev/null +++ b/doc/rsyslog_confgraph_std.conf @@ -0,0 +1,79 @@ +#rsyslog v3 config file + +# if you experience problems, check +# http://www.rsyslog.com/troubleshoot for assistance + +#### MODULES #### + +$ModLoad imuxsock.so # provides support for local system logging (e.g. via logger command) +$ModLoad imklog.so # provides kernel logging support (previously done by rklogd) +#$ModLoad immark.so # provides --MARK-- message capability + +# Provides UDP syslog reception +#$ModLoad imudp.so +#$UDPServerRun 514 + +# Provides TCP syslog reception +#$ModLoad imtcp.so +#$InputTCPServerRun 514 + + +#### GLOBAL DIRECTIVES #### + +# Use default timestamp format +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + +# File syncing capability is disabled by default. This feature is usually not required, +# not useful and an extreme performance hit +#$ActionFileEnableSync on + + +#### RULES #### + +# Log all kernel messages to the console. +# Logging much else clutters up the screen. +#kern.* /dev/console + +# Log anything (except mail) of level info or higher. +# Don't log private authentication messages! +*.info;mail.none;authpriv.none;cron.none /var/log/messages + +# The authpriv file has restricted access. +authpriv.* /var/log/secure + +# Log all the mail messages in one place. +mail.* -/var/log/maillog + + +# Log cron stuff +cron.* /var/log/cron + +# Everybody gets emergency messages +*.emerg * + +# Save news errors of level crit and higher in a special file. +uucp,news.crit /var/log/spooler + +# Save boot messages also to boot.log +local7.* /var/log/boot.log + + + +# ### begin forwarding rule ### +# The statement between the begin ... end define a SINGLE forwarding +# rule. They belong together, do NOT split them. If you create multiple +# forwarding rules, duplicate the whole block! +# Remote Logging (we use TCP for reliable delivery) +# +# An on-disk queue is created for this action. If the remote host is +# down, messages are spooled to disk and sent when it is up again. +#$WorkDirectory /var/spppl/rsyslog # where to place spool files +#$ActionQueueFileName fwdRule1 # unique name prefix for spool files +#$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) +#$ActionQueueSaveOnShutdown on # save messages to disk on shutdown +$ActionQueueType LinkedList # run asynchronously +#$ActionResumeRetryCount -1 # infinite retries if host is down +# remote host is: name/ip:port, e.g. 192.168.0.1:514, port optional +*.* @@remote-host:514 +# ### end of the forwarding rule ### +$GenerateConfigGraph /home/rger/proj/rsyslog/rsyslog.dot diff --git a/doc/rsyslog_confgraph_std.png b/doc/rsyslog_confgraph_std.png Binary files differnew file mode 100644 index 00000000..655a7f82 --- /dev/null +++ b/doc/rsyslog_confgraph_std.png diff --git a/doc/rsyslog_high_database_rate.html b/doc/rsyslog_high_database_rate.html index 158a4df6..2bae58c6 100644 --- a/doc/rsyslog_high_database_rate.html +++ b/doc/rsyslog_high_database_rate.html @@ -7,6 +7,7 @@ </head> <body> +<a href="features.html">back</a> <h1>Handling a massive syslog database insert rate with Rsyslog</h1> @@ -171,6 +172,14 @@ comments or find bugs (I *do* bugs - no way... ;)), please http://www.gnu.org/copyleft/fdl.html</a>.</p> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> </body> diff --git a/doc/rsyslog_mysql.html b/doc/rsyslog_mysql.html index 753c86ec..a27bd59e 100644 --- a/doc/rsyslog_mysql.html +++ b/doc/rsyslog_mysql.html @@ -1,6 +1,6 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head><title>Writing syslog Data to MySQL</title> - +<a href="features.html">back</a> <meta name="KEYWORDS" content="syslog, mysql, syslog to mysql, howto"></head> <body> <h1>Writing syslog messages to MySQL</h1> @@ -259,4 +259,13 @@ document under the terms of the GNU Free Documentation License, Version with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at <a href="http://www.gnu.org/copyleft/fdl.html"> http://www.gnu.org/copyleft/fdl.html</a>.</p> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> + </body></html> diff --git a/doc/rsyslog_ng_comparison.html b/doc/rsyslog_ng_comparison.html index 2f383f78..8e121a8d 100644 --- a/doc/rsyslog_ng_comparison.html +++ b/doc/rsyslog_ng_comparison.html @@ -1,6 +1,7 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head><title>rsyslog vs. syslog-ng - a comparison</title></head> <body> +<a href="features.html">back</a> <h1>rsyslog vs. syslog-ng</h1> <p><small><i>Written by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> (2008-05-06)</i></small></p> @@ -584,4 +585,13 @@ feature sheet. I have not yet been able to fully work through it. In the mean time, you may want to read it in parallel. It is available at <a href="http://www.balabit.com/network-security/syslog-ng/features/detailed/">Balabit's site</a>.</p> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> + </body></html> diff --git a/doc/rsyslog_queue_pointers.jpeg b/doc/rsyslog_queue_pointers.jpeg Binary files differnew file mode 100644 index 00000000..809dd446 --- /dev/null +++ b/doc/rsyslog_queue_pointers.jpeg diff --git a/doc/rsyslog_queue_pointers2.jpeg b/doc/rsyslog_queue_pointers2.jpeg Binary files differnew file mode 100644 index 00000000..2ad60113 --- /dev/null +++ b/doc/rsyslog_queue_pointers2.jpeg diff --git a/doc/rsyslog_stunnel.html b/doc/rsyslog_stunnel.html index 1d024934..f0c0b3af 100644 --- a/doc/rsyslog_stunnel.html +++ b/doc/rsyslog_stunnel.html @@ -1,5 +1,6 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head> +<a href="features.html">back</a> <title>SSL Encrypting syslog with stunnel</title><meta name="KEYWORDS" content="syslog encryption, rsyslog, stunnel, secure syslog, tcp, reliable, howto, ssl"></head><body> <h1>SSL Encrypting Syslog with Stunnel</h1> @@ -236,5 +237,13 @@ comments or find bugs (I *do* bugs - no way... ;)), please Texts. A copy of the license can be viewed at <a href="http://www.gnu.org/copyleft/fdl.html"> http://www.gnu.org/copyleft/fdl.html</a>.</p> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> </body></html> diff --git a/doc/rsyslog_tls.html b/doc/rsyslog_tls.html index a26a9f53..e37d26a7 100644 --- a/doc/rsyslog_tls.html +++ b/doc/rsyslog_tls.html @@ -1,5 +1,6 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html><head><title>TLS (SSL) Encrypting syslog</title> +<a href="features.html">back</a> <meta name="KEYWORDS" content="syslog encryption, rsyslog, secure syslog, tcp, reliable, howto, ssl, tls"> </head> @@ -304,4 +305,13 @@ document under the terms of the GNU Free Documentation License, Version with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at <a href="http://www.gnu.org/copyleft/fdl.html">http://www.gnu.org/copyleft/fdl.html</a>.</p> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> + </body></html> diff --git a/doc/src/dataflow.dia b/doc/src/dataflow.dia Binary files differnew file mode 100644 index 00000000..3875fc61 --- /dev/null +++ b/doc/src/dataflow.dia diff --git a/doc/src/direct_queue0.dia b/doc/src/direct_queue0.dia Binary files differnew file mode 100644 index 00000000..4446619b --- /dev/null +++ b/doc/src/direct_queue0.dia diff --git a/doc/src/direct_queue1.dia b/doc/src/direct_queue1.dia Binary files differnew file mode 100644 index 00000000..7a64ea09 --- /dev/null +++ b/doc/src/direct_queue1.dia diff --git a/doc/src/direct_queue2.dia b/doc/src/direct_queue2.dia Binary files differnew file mode 100644 index 00000000..b0c394c0 --- /dev/null +++ b/doc/src/direct_queue2.dia diff --git a/doc/src/direct_queue3.dia b/doc/src/direct_queue3.dia Binary files differnew file mode 100644 index 00000000..bc477b25 --- /dev/null +++ b/doc/src/direct_queue3.dia diff --git a/doc/src/direct_queue_directq.dia b/doc/src/direct_queue_directq.dia Binary files differnew file mode 100644 index 00000000..37fdb44c --- /dev/null +++ b/doc/src/direct_queue_directq.dia diff --git a/doc/src/direct_queue_rsyslog.dia b/doc/src/direct_queue_rsyslog.dia Binary files differnew file mode 100644 index 00000000..9a030117 --- /dev/null +++ b/doc/src/direct_queue_rsyslog.dia diff --git a/doc/src/direct_queue_rsyslog2.dia b/doc/src/direct_queue_rsyslog2.dia Binary files differnew file mode 100644 index 00000000..c596f39f --- /dev/null +++ b/doc/src/direct_queue_rsyslog2.dia diff --git a/doc/src/queue_analogy_tv.dia b/doc/src/queue_analogy_tv.dia Binary files differnew file mode 100644 index 00000000..00fbdeb5 --- /dev/null +++ b/doc/src/queue_analogy_tv.dia diff --git a/doc/src/rsyslog_queue_pointers.dia b/doc/src/rsyslog_queue_pointers.dia Binary files differnew file mode 100644 index 00000000..2ad4cacb --- /dev/null +++ b/doc/src/rsyslog_queue_pointers.dia diff --git a/doc/src/rsyslog_queue_pointers2.dia b/doc/src/rsyslog_queue_pointers2.dia Binary files differnew file mode 100644 index 00000000..6a35c664 --- /dev/null +++ b/doc/src/rsyslog_queue_pointers2.dia diff --git a/doc/status.html b/doc/status.html index 5c8409ec..56ad78d9 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,29 +2,39 @@ <html><head><title>rsyslog status page</title></head> <body> <h2>rsyslog status page</h2> -<p>This page reflects the status as of 2008-10-22.</p> +<p>This page reflects the status as of 2009-07-08.</p> <h2>Current Releases</h2> -<p><b>development:</b> 3.21.6 [2008-10-22] - -<a href="http://www.rsyslog.com/Article292.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-135.phtml">download</a> +<p><b>v5 development:</b> 5.1.2 [2009-07-08] - +<a href="http://www.rsyslog.com/Article386.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-166.phtml">download</a> -<br><b>beta:</b> 3.19.12 [2008-10-16] - -<a href="http://www.rsyslog.com/Article283.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-134.phtml">download</a></p> +<br><b>v4 development:</b> 4.5.0 [2009-07-03] - +<a href="http://www.rsyslog.com/Article380.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-164.phtml">download</a></p> -<p><b>v3 stable:</b> 3.18.5 [2008-10-09] - <a href="http://www.rsyslog.com/Article281.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-133.phtml">download</a> +<br><b>beta:</b> 4.3.2 [2009-06-24] - +<a href="http://www.rsyslog.com/Article378.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-162.phtml">download</a></p> -<br><b>v2 stable:</b> 2.0.6 [2008-08-07] - <a href="http://www.rsyslog.com/Article266.phtml">change log</a> - -<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-125.phtml">download</a> -<br>v0 and v1 are deprecated and no longer supported. If you absolutely do not like to +<p><b>v4 stable:</b> 4.2.0 [2009-06-23] - +<a href="http://www.rsyslog.com/Article376.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-161.phtml">download</a> + +<br><b>v3 stable:</b> 3.22.1 [2009-07-02] - +<a href="http://www.rsyslog.com/Article381.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-163.phtml">download</a> + +<br>v2 stable: 2.0.7 [2009-04-14] - <a href="http://www.rsyslog.com/Article362.phtml">change log</a> - +<a href="http://www.rsyslog.com/Downloads-req-viewdownloaddetails-lid-154.phtml">download</a> +<br>v0 to v2 are deprecated and no longer supported. If you absolutely do not like to upgrade, you may consider purchasing a <a href="professional_support.html">commercial rsyslog support package</a>. Just let us point out that it is really not a good idea to still run a v0 version. <p><a href="v3compatibility.html">If you updgrade from version 2, be sure to read the rsyslog v3 -compatibility document.</a></p> +compatibility document.</a> There are no additional compatibility concerns at this time for +upgrading from v3 to v4. If some occur, we will post an additional compatiblity document.</p> <p>(<a href="version_naming.html">How are versions named?</a>)</p> <h2>Platforms</h2> diff --git a/doc/syslog_protocol.html b/doc/syslog_protocol.html index 72de5c27..57eb9ffe 100644 --- a/doc/syslog_protocol.html +++ b/doc/syslog_protocol.html @@ -3,6 +3,7 @@ <title>syslog-protocol support in rsyslog</title> </head> <body> +<a href="features.html">back</a> <h1>syslog-protocol support in rsyslog</h1> <p><b><a href="http://www.rsyslog.com/">Rsyslog</a> provides a trial implementation of the proposed @@ -191,6 +192,14 @@ discussed ;)</p> syslog-protocol should be further evaluated and be fully understood</li> </ul> <p> </p> +<p>[<a href="manual.html">manual index</a>] +[<a href="rsyslog_conf.html">rsyslog.conf</a>] +[<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> </body> </html> diff --git a/doc/troubleshoot.html b/doc/troubleshoot.html index e655c2ef..a8855fd4 100644 --- a/doc/troubleshoot.html +++ b/doc/troubleshoot.html @@ -18,7 +18,14 @@ is a rsyslog-doc package, that often needs to be installed separately. <p><b>Malformed Messages and Message Properties</b> <p>A common trouble source are <a href="syslog_parsing.html">ill-formed syslog messages</a>, which lead to to all sorts of interesting problems, including malformed hostnames and dates. -Read the quoted guide to find relief. +Read the quoted guide to find relief. A common symptom is that the %HOSTNAME% property is +used for generating dynafile names, but some glibberish shows up. This is caused by the +malformed syslog messages, so be sure to read the +<a href="syslog_parsing.html">guide</a> if you face that problem. Just let me add that the +common work-around is to use %FROMHOST% or %FROMHOST-IP% instead. These do not take the +hostname from the message, but rather use the host that sent the message (taken from +the socket layer). Of course, this does not work over NAT or relay chains, where the +only cure is to make sure senders emit well-formed messages. <p><b>Configuration Problems</b> <p>Rsyslog 3.21.1 and above has been enhanced to support extended configuration checking. It offers a special command line switch (-N1) that puts it into "config verfication mode". @@ -28,6 +35,15 @@ mode can be used in parallel to a running instance of rsyslogd. <p><b><i>/path/to/rsyslogd -f/path/to/config-file -N1</i></b> <p>You should also specify other options you usually give (like -c3 and whatever else). Any problems experienced are reported to stderr [aka "your screen" (if not redirected)]. +<p><b>Configuration Graphs</b> +<p>Starting with rsyslog 4.3.1, the +"<a href="rsconf1_generateconfiggraph.html">$GenerateConfigGraph</a>" +command is supported, a very valuable troubleshooting tool. It permits to +generate a graph of how rsyslogd understood its configuration file. It is assumed that +many configuration issues can easily be detected just by looking at the configuration graph. +Full details of how to generate the graphs, and what to look for can be found in the +"<a href="rsconf1_generateconfiggraph.html">$GenerateConfigGraph</a>" +manual page. <p><b>Asking for Help</b> <p>If you can't find the answer yourself, you should look at these places for community help. |
