The tokenizer now skips leading terminator characters

diff --git a/include/ulas.h b/include/ulas.h
index e50ca2eed965b9c94898f19d741af1defd350e24..b7ff99bf4ee216dfdc6dbd2a5cc41f9f1439eb0f 100644 (file)
--- a/include/ulas.h
+++ b/include/ulas.h
@@ -127,10 +127,10 @@ char *ulas_strndup(const char *src, size_t n);
  * A token rule returns true when a token should end
  * otherwise returns false
  */
-typedef bool (*ulas_tokrule)(char current, char prev);
+typedef bool (*ulas_tokrule)(char current);
 
 // simple tokenizer at any space char
-bool ulas_tokrulespace(char current, char prev);
+bool ulas_tokrulespace(char current);
 
 // tokenisze according to pre-defined rules
 // returns the amount of bytes of line that were

diff --git a/src/test.c b/src/test.c
index 11eab366c511c7252b28d9935f44cb8ef1591936..da84d42f92d270f5482f6309c697b7022b132dfd 100644 (file)
--- a/src/test.c
+++ b/src/test.c
@@ -17,6 +17,7 @@ void test_tok(void) {
   TESTBEGIN("tok");
 
   assert_tok("test", 4, "test tokens", ulas_tokrulespace);
+  assert_tok("test", 6, "  test tokens", ulas_tokrulespace);
   assert_tok("tokens", 6, "tokens", ulas_tokrulespace);
   assert_tok("", 0, "", ulas_tokrulespace);
   assert_tok("", -1, NULL, ulas_tokrulespace);

diff --git a/src/ulas.c b/src/ulas.c
index b060a9eb7c3ca041bad5419ed589ce2e6d1e83a2..6adbe18d43c96b4485d588364215008a3432b582 100644 (file)
--- a/src/ulas.c
+++ b/src/ulas.c
@@ -56,7 +56,7 @@ int ulas_main(struct ulas_config cfg) {
   return 0;
 }
 
-bool ulas_tokrulespace(char current, char prev) { return isspace(current); }
+bool ulas_tokrulespace(char current) { return isspace(current); }
 
 int ulas_tok(char *dst, const char *line, size_t n, ulas_tokrule rule) {
   if (!dst || !line || n == 0) {
@@ -64,17 +64,22 @@ int ulas_tok(char *dst, const char *line, size_t n, ulas_tokrule rule) {
   }
 
   int i = 0;
-  char prev = '\0';
+  int write = 0;
   char current = '\0';
-  for (i = 0; i < n - 1 && line[i]; i++) {
-    prev = current;
+
+  // always skip leading terminators
+  while (line[i] && i < n - 1 && rule(line[i])) {
+    i++;
+  }
+
+  for (; i < n - 1 && write < n - 1 && line[i]; i++, write++) {
     current = line[i];
-    if (rule(current, prev)) {
+    if (rule(current)) {
       break;
     }
-    dst[i] = current;
+    dst[write] = current;
   }
 
-  dst[i + 1] = '\0';
+  dst[write + 1] = '\0';
   return i;
 }