From 5310d45d294db4b992dfcab311af83d42bdcfea9 Mon Sep 17 00:00:00 2001
From: Lukas Krickl <lukas@krickl.dev>
Date: Thu, 9 Nov 2023 15:15:23 +0100
Subject: [PATCH] Refactored tokenizer to allow for terminator tokens

---
 src/test.c |  6 +++---
 src/ulas.c | 60 ++++++++++++++++++++++++++++++------------------------
 2 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/src/test.c b/src/test.c
index 88cf57a..8b219a1 100644
--- a/src/test.c
+++ b/src/test.c
@@ -31,9 +31,9 @@
 void test_tok(void) {
   TESTBEGIN("tok");
 
-  assert_tok("  test  tokens   with   line / * + - , ; $1",
-             {"test", "tokens", "with", "line", "/", "*", "+", "-", ",", ";",
-              "$1", NULL});
+  assert_tok("  test  tokens   with,   line / * + - , ; $1",
+             {"test", "tokens", "with", ",", "line", "/", "*", "+", "-", ",",
+              ";", "$1", NULL});
 
   TESTEND("tok");
 }
diff --git a/src/ulas.c b/src/ulas.c
index 3e685d0..e749258 100644
--- a/src/ulas.c
+++ b/src/ulas.c
@@ -86,39 +86,45 @@ int ulas_tok(struct ulas_str *dst, const char **out_line, size_t n) {
     i++;
   }
 
-  char c = line[i];
-
-  switch (c) {
-  case ',':
-  case '+':
-  case '-':
-  case '*':
-  case '/':
-  case '\\':
-  case ULAS_TOK_COMMENT:
-    // single char tokens
-    dst->buf[write++] = line[i++];
-    break;
-  case '$':
-    // special var for preprocessor
-    // make sure we have enough space in buffer
-    ulas_strensr(dst, write + 2);
-    // escape char tokens
-    dst->buf[write++] = line[i++];
-    dst->buf[write++] = line[i++];
-    break;
-  default:
-    while (weld_tokcond) {
+  while (weld_tokcond) {
+    char c = line[i];
+
+    switch (c) {
+    case ',':
+    case '+':
+    case '-':
+    case '*':
+    case '/':
+    case '\\':
+    case ULAS_TOK_COMMENT:
+      if (write) {
+        goto tokdone;
+      }
+      // single char tokens
+      dst->buf[write++] = line[i++];
+      goto tokdone;
+    case '$':
+      if (write) {
+        goto tokdone;
+      }
+      // special var for preprocessor
+      // make sure we have enough space in buffer
+      ulas_strensr(dst, write + 2);
+      // escape char tokens
+      dst->buf[write++] = line[i++];
+      dst->buf[write++] = line[i++];
+      goto tokdone;
+    default:
       if (isspace(line[i])) {
-        break;
+        goto tokdone;
       }
       dst->buf[write] = line[i];
-      i++;
       write++;
+      break;
     }
-    break;
+    i++;
   }
-
+tokdone:
 #undef weld_tokcond
 
   dst->buf[write] = '\0';
-- 
2.30.2