Refactored tokenizer to allow for terminator tokens

diff --git a/src/test.c b/src/test.c
index 88cf57acbda42fb6b1f6fed1f94a6359e46d417b..8b219a1acc5fa0bf113d58c60af52b8426f76746 100644 (file)
--- a/src/test.c
+++ b/src/test.c
@@ -31,9 +31,9 @@
 void test_tok(void) {
   TESTBEGIN("tok");
 
-  assert_tok("  test  tokens   with   line / * + - , ; $1",
-             {"test", "tokens", "with", "line", "/", "*", "+", "-", ",", ";",
-              "$1", NULL});
+  assert_tok("  test  tokens   with,   line / * + - , ; $1",
+             {"test", "tokens", "with", ",", "line", "/", "*", "+", "-", ",",
+              ";", "$1", NULL});
 
   TESTEND("tok");
 }

diff --git a/src/ulas.c b/src/ulas.c
index 3e685d04f8b3a1555054555352564891546e978d..e749258abd626855a4ba8ec8bd130e156ff344c3 100644 (file)
--- a/src/ulas.c
+++ b/src/ulas.c
@@ -86,39 +86,45 @@ int ulas_tok(struct ulas_str *dst, const char **out_line, size_t n) {
     i++;
   }
 
-  char c = line[i];
-
-  switch (c) {
-  case ',':
-  case '+':
-  case '-':
-  case '*':
-  case '/':
-  case '\\':
-  case ULAS_TOK_COMMENT:
-    // single char tokens
-    dst->buf[write++] = line[i++];
-    break;
-  case '$':
-    // special var for preprocessor
-    // make sure we have enough space in buffer
-    ulas_strensr(dst, write + 2);
-    // escape char tokens
-    dst->buf[write++] = line[i++];
-    dst->buf[write++] = line[i++];
-    break;
-  default:
-    while (weld_tokcond) {
+  while (weld_tokcond) {
+    char c = line[i];
+
+    switch (c) {
+    case ',':
+    case '+':
+    case '-':
+    case '*':
+    case '/':
+    case '\\':
+    case ULAS_TOK_COMMENT:
+      if (write) {
+        goto tokdone;
+      }
+      // single char tokens
+      dst->buf[write++] = line[i++];
+      goto tokdone;
+    case '$':
+      if (write) {
+        goto tokdone;
+      }
+      // special var for preprocessor
+      // make sure we have enough space in buffer
+      ulas_strensr(dst, write + 2);
+      // escape char tokens
+      dst->buf[write++] = line[i++];
+      dst->buf[write++] = line[i++];
+      goto tokdone;
+    default:
       if (isspace(line[i])) {
-        break;
+        goto tokdone;
       }
       dst->buf[write] = line[i];
-      i++;
       write++;
+      break;
     }
-    break;
+    i++;
   }
-
+tokdone:
 #undef weld_tokcond
 
   dst->buf[write] = '\0';