From: Lukas Krickl <lukas@krickl.dev>
Date: Tue, 5 Dec 2023 09:25:53 +0000 (+0100)
Subject: Fixed string tokenizer
X-Git-Url: https://git.krickl.dev/?a=commitdiff_plain;h=e8c2dc4efaec921399d76158a80db32a097ed1fa;p=ulas%2F.git

Fixed string tokenizer
---

diff --git a/src/test.c b/src/test.c
index 6e4aff3..92e6561 100644
--- a/src/test.c
+++ b/src/test.c
@@ -295,7 +295,8 @@ void test_strexpr(void) {
   TESTBEGIN("strexpr");
 
   ASSERT_STREXPR("test", 0, "\"test\"");
-  // ASSERT_STREXPR("test\n", 0, "\"test\\n\"");
+  ASSERT_STREXPR("test\"", 0, "\"test\\\"\"");
+  ASSERT_STREXPR("test\" 123", 0, "\"test\\\" 123\"");
 
   TESTEND("strexpr");
 }
diff --git a/src/ulas.c b/src/ulas.c
index eabccfa..b2464c5 100644
--- a/src/ulas.c
+++ b/src/ulas.c
@@ -288,69 +288,81 @@ int ulas_tok(struct ulas_str *dst, const char **out_line, unsigned long n) {
     i++;
   }
 
-  while (WELD_TOKCOND) {
-    char c = line[i];
-
-    switch (c) {
-    case '+':
-    case '-':
-    case '*':
-    case '/':
-    case '~':
-    case '|':
-    case '&':
-    case '%':
-    case '(':
-    case ')':
-    case '[':
-    case ']':
-    case ',':
-    case '\\':
-    case ULAS_TOK_COMMENT:
-      if (WELD_TOKISTERM) {
-        goto tokdone;
-      }
-      // single char tokens
-      dst->buf[write++] = line[i++];
-      goto tokdone;
-    case '$':
-      if (WELD_TOKISTERM) {
+  // string token
+  if (line[i] == '"') {
+    dst->buf[write++] = line[i++];
+    int last_escape = 0;
+    while (WELD_TOKCOND && (line[i] != '\"' || last_escape)) {
+      last_escape = line[i] == '\\';
+      dst->buf[write++] = line[i];
+      i++;
+    }
+    dst->buf[write++] = line[i++];
+  } else {
+    while (WELD_TOKCOND) {
+      char c = line[i];
+
+      switch (c) {
+      case '+':
+      case '-':
+      case '*':
+      case '/':
+      case '~':
+      case '|':
+      case '&':
+      case '%':
+      case '(':
+      case ')':
+      case '[':
+      case ']':
+      case ',':
+      case '\\':
+      case ULAS_TOK_COMMENT:
+        if (WELD_TOKISTERM) {
+          goto tokdone;
+        }
+        // single char tokens
+        dst->buf[write++] = line[i++];
         goto tokdone;
-      }
-      // special var for preprocessor
-      // make sure we have enough space in buffer
-      ulas_strensr(dst, write + 2);
-      // escape char tokens
-      dst->buf[write++] = line[i++];
-      if (line[i] && !isspace(line[i])) {
+      case '$':
+        if (WELD_TOKISTERM) {
+          goto tokdone;
+        }
+        // special var for preprocessor
+        // make sure we have enough space in buffer
+        ulas_strensr(dst, write + 2);
+        // escape char tokens
         dst->buf[write++] = line[i++];
-      }
-      goto tokdone;
-    case '=':
-    case '<':
-    case '!':
-    case '>':
-      if (line[i + 1] == '=') {
+        if (line[i] && !isspace(line[i])) {
+          dst->buf[write++] = line[i++];
+        }
+        goto tokdone;
+      case '=':
+      case '<':
+      case '!':
+      case '>':
+        if (line[i + 1] == '=') {
+          dst->buf[write] = line[i];
+          i++;
+          write++;
+        }
         dst->buf[write] = line[i];
+        write++;
         i++;
+        goto tokdone;
+      default:
+        if (isspace(line[i])) {
+          goto tokdone;
+        }
+        dst->buf[write] = line[i];
         write++;
+        break;
       }
-      dst->buf[write] = line[i];
-      write++;
       i++;
-      goto tokdone;
-    default:
-      if (isspace(line[i])) {
-        goto tokdone;
-      }
-      dst->buf[write] = line[i];
-      write++;
-      break;
     }
-    i++;
   }
 tokdone:
-  
+
   dst->buf[write] = '\0';
 
   *out_line += i;