Fix tokenize function and test.

Previous implementation was error-prone, because allowed returning empty
tokens (mimiced strsep()), which is fine if there is a NULL semantic.
Unfortunately SystemTap doesn't provide it in scripts and has only blank
string (""), therefore testing against it was misleading.
The solution is to return only non-empty tokens (mimic strtok()).

* tapset/string.stp: Fix tokenize.
* testsuite/systemtap.string/tokenize.stp: Improve and add case with
  more than one delimiter in the delim string.
* testsuite/systemtap.string/tokenize.exp: Ditto.
* stapfuncs.3stap.in: Update tokenize description.
* doc/langref.tex: Ditto.

Signed-off-by: Josh Stone <jistone@redhat.com>

1 files changed, 47 insertions, 28 deletions

diff --git a/testsuite/systemtap.string/tokenize.stp b/testsuite/systemtap.string/tokenize.stp
index 10703d90..1b253c8d 100644
--- a/testsuite/systemtap.string/tokenize.stp
+++ b/testsuite/systemtap.string/tokenize.stp
@@ -5,42 +5,61 @@ probe begin
 	teststr3 = "1,,2,3, ,4"
 	teststr4 = ""
 	teststr5 = "this is a string with no delimiters"
+	teststr6 = "this is a string, which has two delimiters"
 
 	tok = tokenize(teststr1, "|")
 	while (tok != "") {
-		printf("%s\n", tok)
+		println(tok)
 		tok = tokenize("", "|")
 	}
 
-        tok = tokenize(teststr1, ",")
-        while (tok != "") {
-                printf("%s\n", tok)
-                tok = tokenize("", "|")
-        }
+	println("-")
+
+	tok = tokenize(teststr1, ",")
+	while (tok != "") {
+		println(tok)
+		tok = tokenize("", "|")
+	}
 	
+	println("-")
+
 	tok = tokenize(teststr2, ",")
-        while (tok != "") {
-                printf("%s\n", tok)
-                tok = tokenize("", ",")
-        }
-
-        tok = tokenize(teststr3, ",")
-        while (tok != "") {
-                printf("%s\n", tok)
-                tok = tokenize("", ",")
-        }
-
-        tok = tokenize(teststr4, ",")
-        while (tok != "") {
-                printf("%s\n", tok)
-                tok = tokenize("", ",")
-        }
-
-        tok = tokenize(teststr5, ",")
-        while (tok != "") {
-                printf("%s\n", tok)
-                tok = tokenize("", ",")
-        }
+	while (tok != "") {
+		println(tok)
+		tok = tokenize("", ",")
+	}
+
+	println("-")
+
+	tok = tokenize(teststr3, ",")
+	while (tok != "") {
+		println(tok)
+		tok = tokenize("", ",")
+	}
+
+	println("-")
+
+	tok = tokenize(teststr4, ",")
+	while (tok != "") {
+		println(tok)
+		tok = tokenize("", ",")
+	}
+
+	println("-")
+
+	tok = tokenize(teststr5, ",")
+	while (tok != "") {
+		println(tok)
+		tok = tokenize("", ",")
+	}
+
+	println("-")
+
+	tok = tokenize(teststr6, ", ")
+	while (tok != "") {
+		println(tok)
+		tok = tokenize("", ", ")
+	}
 
 	exit()
 }