- modified parser and tokenizer to support slight ABNF modifications from

    yesterday
- change in ABNF was wrong - made a slightly different change
- fixed bugs in tokenizer
- expression compiler finished (except bugs, of course ;))

7 files changed, 86 insertions, 30 deletions

diff --git a/ctok.c b/ctok.c
index db5609ec..8b88e1a2 100644
--- a/ctok.c
+++ b/ctok.c
@@ -224,6 +224,10 @@ ctokGetNumber(ctok_t *pThis, ctok_token_t *pToken)
 		CHKiRet(ctokGetCharFromStream(pThis, &c));
 		c = tolower(c);
 	}
+
+	/* we need to unget the character that made the loop terminate */
+	CHKiRet(ctokUngetCharFromStream(pThis, c));
+
 	CHKiRet(var.SetInt64(pToken->pVar, n));
 
 dbgprintf("number, number is: '%lld'\n", n);
@@ -518,7 +522,7 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken)
 	} while(bRetry); /* warning: do ... while()! */
 
 	*ppToken = pToken;
-RUNLOG_VAR("%d", pToken->tok);
+	dbgoprint((obj_t*) pToken, "token: %d\n", pToken->tok);
 
 finalize_it:
 	if(iRet != RS_RET_OK) {
diff --git a/debug.c b/debug.c
index b88e2ec6..00b7916b 100644
--- a/debug.c
+++ b/debug.c
@@ -930,7 +930,7 @@ int dbgEntrFunc(dbgFuncDB_t *pFuncDB, int line)
 	/* when we reach this point, we have a fully-initialized FuncDB! */
 
 	//if(bLogFuncFlow) /* quick debug hack...  select the best for you! */
-	if(bLogFuncFlow && !strcmp((char*)pFuncDB->file, "omlibdbi.c")) /* quick debug hack... select the best for you! */
+	if(bLogFuncFlow && !strcmp((char*)pFuncDB->file, "expr.c")) /* quick debug hack... select the best for you! */
 		dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func);
 	if(pThrd->stackPtr >= (int) (sizeof(pThrd->callStack) / sizeof(dbgFuncDB_t*))) {
 		dbgprintf("%s:%d: %s: debug module: call stack for this thread full, suspending call tracking\n",
@@ -960,7 +960,7 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore)
 
 	dbgFuncDBPrintActiveMutexes(pFuncDB, "WARNING: mutex still owned by us as we exit function, mutex: ", pthread_self());
 	//if(bLogFuncFlow) /* quick debug hack... select the best for you! */
-	if(bLogFuncFlow && !strcmp((char*)pFuncDB->file, "omlibdbi.c")) /* quick debug hack... select the best for you! */
+	if(bLogFuncFlow && !strcmp((char*)pFuncDB->file, "expr.c")) /* quick debug hack... select the best for you! */
 		dbgprintf("%s:%d: %s: exit\n", pFuncDB->file, pFuncDB->line, pFuncDB->func);
 	pThrd->stackPtr = iStackPtrRestore;
 	if(pThrd->stackPtr < 0) {
diff --git a/doc/expression.html b/doc/expression.html
index b2ed7a06..b10b5c3d 100644
--- a/doc/expression.html
+++ b/doc/expression.html
@@ -9,7 +9,7 @@ far, they are supported for filtering messages.</p><p>C-like comments (/* some c
 <h2>Formal Definition</h2>
 
 <p>Below is the formal definition of expression format (in ABNF, RFC 2234):<br>
-</p><pre>if_stmt  := "if" expr "then"  # an aid, not part of expression itself<br><br>expr     := e_and *("or" e_and)<br>e_and    := e_cmp *("and" e_cmp)<br>e_cmp    := val 0*1(cmp_op val)<br>val      := ["-"] term *(("+" / "-") term)<br>term     := factor *(("*" / "/" / "%") factor)<br>factor   := ["not"] terminal<br>terminal := var / constant / function / ( "(" expr ")" )<br>function := name "(" *("," expr) ")"<br>var      := "$" varname<br>varname  := msgvar / sysvar<br>msgvar   := name<br>sysvar   := "$" name<br>name     := alpha *(alnum)<br>constant := string / number<br>string   := simpstr / tplstr ; tplstr will be implemented in next phase<br>simpstr  := "'" *char "'"    ; use your imagination for char ;)<br>tplstr   := '"' template '"' ; not initially implemented<br>number   := ["-"] 1*digit ; 0nn = octal, 0xnn = hex, nn = decimal<br>cmp_op   := "==" / "!=" / "&lt;&gt;" / "&lt;" / "&gt;" / "&lt;=" / "&gt;=" / "contains" / "startswith"<br>digit    := %x30-39<br>alpha    := "a" ... "z" # all letters<br>alnum    :* alpha / digit / "_"<br></pre>
+</p><pre>if_stmt  := "if" expr "then"  # an aid, not part of expression itself<br><br>expr     := e_and *("or" e_and)<br>e_and    := e_cmp *("and" e_cmp)<br>e_cmp    := val 0*1(cmp_op val)<br>val      := term *(("+" / "-") term)<br>term     := factor *(("*" / "/" / "%") factor)<br>factor   := ["not"] ["-"] terminal<br>terminal := var / constant / function / ( "(" expr ")" )<br>function := name "(" *("," expr) ")"<br>var      := "$" varname<br>varname  := msgvar / sysvar<br>msgvar   := name<br>sysvar   := "$" name<br>name     := alpha *(alnum)<br>constant := string / number<br>string   := simpstr / tplstr ; tplstr will be implemented in next phase<br>simpstr  := "'" *char "'"    ; use your imagination for char ;)<br>tplstr   := '"' template '"' ; not initially implemented<br>number   := ["-"] 1*digit ; 0nn = octal, 0xnn = hex, nn = decimal<br>cmp_op   := "==" / "!=" / "&lt;&gt;" / "&lt;" / "&gt;" / "&lt;=" / "&gt;=" / "contains" / "startswith"<br>digit    := %x30-39<br>alpha    := "a" ... "z" # all letters<br>alnum    :* alpha / digit / "_"<br></pre>
 <p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>]
 [<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
 <p><font size="2">This documentation is part of the
diff --git a/expr.c b/expr.c
index e84eab17..156621c1 100644
--- a/expr.c
+++ b/expr.c
@@ -62,13 +62,14 @@ static rsRetVal
 terminal(expr_t *pThis, ctok_t *tok)
 {
 	DEFiRet;
-	ctok_token_t *pToken;
+	ctok_token_t *pToken = NULL;
 	var_t *pVar;
 
 	ISOBJ_TYPE_assert(pThis, expr);
 	ISOBJ_TYPE_assert(tok, ctok);
 
 	CHKiRet(ctok.GetToken(tok, &pToken));
+	/* note: pToken is destructed in finalize_it */
 
 	switch(pToken->tok) {
 		case ctok_SIMPSTR:
@@ -78,8 +79,8 @@ terminal(expr_t *pThis, ctok_t *tok)
 			break;
 		case ctok_NUMBER:
 			dbgoprint((obj_t*) pThis, "number\n");
-			CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHCONSTANT, NULL)); /* add to program */
-			// push val
+			CHKiRet(ctok_token.UnlinkVar(pToken, &pVar));
+			CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHCONSTANT, pVar)); /* add to program */
 			break;
 		case ctok_FUNCTION:
 			dbgoprint((obj_t*) pThis, "function\n");
@@ -88,13 +89,13 @@ terminal(expr_t *pThis, ctok_t *tok)
 			break;
 		case ctok_MSGVAR:
 			dbgoprint((obj_t*) pThis, "MSGVAR\n");
-			CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHMSGVAR, NULL)); /* add to program */
-			// push val
+			CHKiRet(ctok_token.UnlinkVar(pToken, &pVar));
+			CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHMSGVAR, pVar)); /* add to program */
 			break;
 		case ctok_SYSVAR:
 			dbgoprint((obj_t*) pThis, "SYSVAR\n");
-			CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHSYSVAR, NULL)); /* add to program */
-			// push val
+			CHKiRet(ctok_token.UnlinkVar(pToken, &pVar));
+			CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHSYSVAR, pVar)); /* add to program */
 			break;
 		case ctok_LPAREN:
 			dbgoprint((obj_t*) pThis, "expr\n");
@@ -103,8 +104,6 @@ terminal(expr_t *pThis, ctok_t *tok)
 			CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one */
 			if(pToken->tok != ctok_RPAREN)
 				ABORT_FINALIZE(RS_RET_SYNTAX_ERROR);
-			CHKiRet(ctok_token.Destruct(&pToken)); /* "eat" processed token */
-			dbgoprint((obj_t*) pThis, "end expr, rparen eaten\n");
 			break;
 		default:
 			dbgoprint((obj_t*) pThis, "invalid token %d\n", pToken->tok);
@@ -113,6 +112,10 @@ terminal(expr_t *pThis, ctok_t *tok)
 	}
 
 finalize_it:
+	if(pToken != NULL) {
+		CHKiRet(ctok_token.Destruct(&pToken)); /* "eat" processed token */
+	}
+
 	RETiRet;
 }
 
@@ -121,6 +124,8 @@ factor(expr_t *pThis, ctok_t *tok)
 {
 	DEFiRet;
 	ctok_token_t *pToken;
+	int bWasNot;
+	int bWasUnaryMinus;
 
 	ISOBJ_TYPE_assert(pThis, expr);
 	ISOBJ_TYPE_assert(tok, ctok);
@@ -128,14 +133,35 @@ factor(expr_t *pThis, ctok_t *tok)
 	CHKiRet(ctok.GetToken(tok, &pToken));
 	if(pToken->tok == ctok_NOT) {
 		dbgprintf("not\n");
+		bWasNot = 1;
+		CHKiRet(ctok_token.Destruct(&pToken)); /* no longer needed */
+		CHKiRet(ctok.GetToken(tok, &pToken)); /* get new one for next check */
+	} else {
+		bWasNot = 0;
+	}
+
+	if(pToken->tok == ctok_MINUS) {
+		dbgprintf("unary minus\n");
+		bWasUnaryMinus = 1;
 		CHKiRet(ctok_token.Destruct(&pToken)); /* no longer needed */
-		CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one */
 	} else {
+		bWasUnaryMinus = 0;
 		/* we could not process the token, so push it back */
 		CHKiRet(ctok.UngetToken(tok, pToken));
 	}
+
 	CHKiRet(terminal(pThis, tok));
-	// vm: not
+
+	/* warning: the order if the two following ifs is important. Do not change them, this
+	 * would change the semantics of the expression!
+	 */
+	if(bWasUnaryMinus) {
+		CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_UNARY_MINUS, NULL)); /* add to program */
+	}
+
+	if(bWasNot == 1) {
+		CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_NOT, NULL)); /* add to program */
+	}
 
 finalize_it:
 	RETiRet;
@@ -146,12 +172,27 @@ static rsRetVal
 term(expr_t *pThis, ctok_t *tok)
 {
 	DEFiRet;
+	ctok_token_t *pToken;
 
 	ISOBJ_TYPE_assert(pThis, expr);
 	ISOBJ_TYPE_assert(tok, ctok);
 
 	CHKiRet(factor(pThis, tok));
-	// vm: +/-
+
+ 	/* *(("*" / "/" / "%") factor) part */
+	CHKiRet(ctok.GetToken(tok, &pToken));
+	while(pToken->tok == ctok_TIMES || pToken->tok == ctok_DIV || pToken->tok == ctok_MOD) {
+		dbgoprint((obj_t*) pThis, "/,*,%%\n");
+		CHKiRet(factor(pThis, tok));
+		CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, (opcode_t) pToken->tok, NULL)); /* add to program */
+		CHKiRet(ctok_token.Destruct(&pToken)); /* no longer needed */
+		CHKiRet(ctok.GetToken(tok, &pToken));
+	}
+
+	/* unget the token that made us exit the loop - it's obviously not one
+	 * we can process.
+	 */
+	CHKiRet(ctok.UngetToken(tok, pToken)); 
 
 finalize_it:
 	RETiRet;
@@ -166,19 +207,22 @@ val(expr_t *pThis, ctok_t *tok)
 	ISOBJ_TYPE_assert(pThis, expr);
 	ISOBJ_TYPE_assert(tok, ctok);
 
+	CHKiRet(term(pThis, tok));
+
+ 	/* *(("+" / "-") term) part */
 	CHKiRet(ctok.GetToken(tok, &pToken));
-	if(pToken->tok == ctok_PLUS || pToken->tok == ctok_MINUS) {
-		/* TODO: this must be a loop! */
-		dbgprintf("plus/minus\n");
+	while(pToken->tok == ctok_PLUS || pToken->tok == ctok_MINUS) {
+		dbgoprint((obj_t*) pThis, "+/-\n");
+		CHKiRet(term(pThis, tok));
+		CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, (opcode_t) pToken->tok, NULL)); /* add to program */
 		CHKiRet(ctok_token.Destruct(&pToken)); /* no longer needed */
-		// vm: +/-???
-		CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one */
-	} else {
-		/* we could not process the token, so push it back */
-		CHKiRet(ctok.UngetToken(tok, pToken));
+		CHKiRet(ctok.GetToken(tok, &pToken));
 	}
 
-	CHKiRet(term(pThis, tok));
+	/* unget the token that made us exit the loop - it's obviously not one
+	 * we can process.
+	 */
+	CHKiRet(ctok.UngetToken(tok, pToken)); 
 
 finalize_it:
 	RETiRet;
@@ -229,10 +273,9 @@ e_and(expr_t *pThis, ctok_t *tok)
 	CHKiRet(ctok.GetToken(tok, &pToken));
 	while(pToken->tok == ctok_AND) {
 		dbgoprint((obj_t*) pThis, "and\n");
-		/* fill structure */
-		CHKiRet(ctok_token.Destruct(&pToken)); /* no longer needed */
 		CHKiRet(e_cmp(pThis, tok));
 		CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_AND, NULL)); /* add to program */
+		CHKiRet(ctok_token.Destruct(&pToken)); /* no longer needed */
 		CHKiRet(ctok.GetToken(tok, &pToken));
 	}
 
@@ -260,11 +303,10 @@ expr(expr_t *pThis, ctok_t *tok)
  	/* *("or" e_and) part */
 	CHKiRet(ctok.GetToken(tok, &pToken));
 	while(pToken->tok == ctok_OR) {
-		/* fill structure */
 		dbgoprint((obj_t*) pThis, "found OR\n");
-		CHKiRet(ctok_token.Destruct(&pToken)); /* no longer needed */
 		CHKiRet(e_and(pThis, tok));
 		CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_OR, NULL)); /* add to program */
+		CHKiRet(ctok_token.Destruct(&pToken)); /* no longer needed */
 		CHKiRet(ctok.GetToken(tok, &pToken));
 	}
 
diff --git a/var.c b/var.c
index e199a0ee..9947f159 100644
--- a/var.c
+++ b/var.c
@@ -78,6 +78,12 @@ CODESTARTobjDebugPrint(var)
 		case VARTYPE_CSTR:
 			dbgoprint((obj_t*) pThis, "type: cstr, val '%s'\n", rsCStrGetSzStr(pThis->val.vpCStr));
 			break;
+		case VARTYPE_INT64:
+			dbgoprint((obj_t*) pThis, "type: int64, val %lld\n", pThis->val.vInt64);
+			break;
+		case VARTYPE_INT:
+			dbgoprint((obj_t*) pThis, "type: int64, val %d\n", pThis->val.vInt);
+			break;
 		default:
 			dbgoprint((obj_t*) pThis, "type %d currently not suppored in debug output\n", pThis->varType);
 			break;
diff --git a/vmop.c b/vmop.c
index 34d70fba..56112be5 100644
--- a/vmop.c
+++ b/vmop.c
@@ -168,7 +168,10 @@ vmopOpcode2Str(vmop_t *pThis, uchar **ppName)
 			*ppName = (uchar*) "PUSHCONSTANT";
 			break;
 		case opcode_POP:
-			*ppName = (uchar*) "";
+			*ppName = (uchar*) "POP";
+			break;
+		case opcode_UNARY_MINUS:
+			*ppName = (uchar*) "UNARY_MINUS";
 			break;
 		default:
 			*ppName = (uchar*) "INVALID opcode";
diff --git a/vmop.h b/vmop.h
index dea54dbd..1ea531b3 100644
--- a/vmop.h
+++ b/vmop.h
@@ -51,6 +51,7 @@ typedef enum {	 /* do NOT start at 0 to detect uninitialized types after calloc(
 	opcode_PUSHSYSVAR   = 1001,	 /* requires var operand */
 	opcode_PUSHMSGVAR   = 1002,	 /* requires var operand */
 	opcode_PUSHCONSTANT = 1003,	 /* requires var operand */
+	opcode_UNARY_MINUS  = 1010
 } opcode_t;