Fix tokenize function and test.

Previous implementation was error-prone, because allowed returning empty
tokens (mimiced strsep()), which is fine if there is a NULL semantic.
Unfortunately SystemTap doesn't provide it in scripts and has only blank
string (""), therefore testing against it was misleading.
The solution is to return only non-empty tokens (mimic strtok()).

* tapset/string.stp: Fix tokenize.
* testsuite/systemtap.string/tokenize.stp: Improve and add case with
  more than one delimiter in the delim string.
* testsuite/systemtap.string/tokenize.exp: Ditto.
* stapfuncs.3stap.in: Update tokenize description.
* doc/langref.tex: Ditto.

Signed-off-by: Josh Stone <jistone@redhat.com>

1 files changed, 15 insertions, 8 deletions

diff --git a/tapset/string.stp b/tapset/string.stp
index 35ee9fa2..cc842929 100644
--- a/tapset/string.stp
+++ b/tapset/string.stp
@@ -70,25 +70,32 @@ function text_strn:string(input:string, len:long, quoted:long)
 
 /*
  * tokenize - Given a string and a token delimiter,
- *            return the next token in the string
- * input  String to tokenize. If NULL, returns the next token in the
- *        string passed in the previous call to tokenize().
- * delim  Token delimiter. Note this is a string, but only the first
- *        character is used as the delimiter.
+ *            return the next non-empty token in the string
+ *            or blank when no more non-empty tokens are left
+ * input  String to tokenize. If NULL, returns the next non-empty token
+ *        in the string passed in the previous call to tokenize().
+ * delim  Token delimiter. Set of characters that delimit the tokens.
  */
 function tokenize:string(input:string, delim:string)
 %{ /* pure */
 	static char str[MAXSTRINGLEN];
 	static char *str_start;
+	static char *str_end;
 	char *token = NULL;
+	char *token_end = NULL;
 
 	if (THIS->input[0]) {
 		strncpy(str, THIS->input, MAXSTRINGLEN);
 		str_start = &str[0];
+		str_end = &str[0] + strlen(str);
+	}
+	do {
+		token = strsep(&str_start, THIS->delim);
+	} while (token && !token[0]);
+	if (token) {
+		token_end = (str_start ? str_start - 1 : str_end);
+		memcpy(THIS->__retvalue, token, token_end - token + 1);
 	}
-	token = strsep(&str_start, THIS->delim);
-	if (token)
-		strncpy (THIS->__retvalue, token, MAXSTRINGLEN);
 %}
 
 /*