Fixed string tokenizer

diff --git a/src/test.c b/src/test.c
index 6e4aff338c2130bd3b6105acc616962f0aa39a50..92e65615cb5ede382ed5cd8a38b7eeea8495d5f1 100644 (file)
--- a/src/test.c
+++ b/src/test.c
@@ -295,7 +295,8 @@ void test_strexpr(void) {
   TESTBEGIN("strexpr");
 
   ASSERT_STREXPR("test", 0, "\"test\"");
-  // ASSERT_STREXPR("test\n", 0, "\"test\\n\"");
+  ASSERT_STREXPR("test\"", 0, "\"test\\\"\"");
+  ASSERT_STREXPR("test\" 123", 0, "\"test\\\" 123\"");
 
   TESTEND("strexpr");
 }

diff --git a/src/ulas.c b/src/ulas.c
index eabccfab94d8077a037d7bdc1d7a35ac53b4f1f2..b2464c592cd76e13eeb135d5e07c8bfb13c4e2a5 100644 (file)
--- a/src/ulas.c
+++ b/src/ulas.c
@@ -288,69 +288,81 @@ int ulas_tok(struct ulas_str *dst, const char **out_line, unsigned long n) {
     i++;
   }
 
-  while (WELD_TOKCOND) {
-    char c = line[i];
-
-    switch (c) {
-    case '+':
-    case '-':
-    case '*':
-    case '/':
-    case '~':
-    case '|':
-    case '&':
-    case '%':
-    case '(':
-    case ')':
-    case '[':
-    case ']':
-    case ',':
-    case '\\':
-    case ULAS_TOK_COMMENT:
-      if (WELD_TOKISTERM) {
-        goto tokdone;
-      }
-      // single char tokens
-      dst->buf[write++] = line[i++];
-      goto tokdone;
-    case '$':
-      if (WELD_TOKISTERM) {
+  // string token
+  if (line[i] == '"') {
+    dst->buf[write++] = line[i++];
+    int last_escape = 0;
+    while (WELD_TOKCOND && (line[i] != '\"' || last_escape)) {
+      last_escape = line[i] == '\\';
+      dst->buf[write++] = line[i];
+      i++;
+    }
+    dst->buf[write++] = line[i++];
+  } else {
+    while (WELD_TOKCOND) {
+      char c = line[i];
+
+      switch (c) {
+      case '+':
+      case '-':
+      case '*':
+      case '/':
+      case '~':
+      case '|':
+      case '&':
+      case '%':
+      case '(':
+      case ')':
+      case '[':
+      case ']':
+      case ',':
+      case '\\':
+      case ULAS_TOK_COMMENT:
+        if (WELD_TOKISTERM) {
+          goto tokdone;
+        }
+        // single char tokens
+        dst->buf[write++] = line[i++];
         goto tokdone;
-      }
-      // special var for preprocessor
-      // make sure we have enough space in buffer
-      ulas_strensr(dst, write + 2);
-      // escape char tokens
-      dst->buf[write++] = line[i++];
-      if (line[i] && !isspace(line[i])) {
+      case '$':
+        if (WELD_TOKISTERM) {
+          goto tokdone;
+        }
+        // special var for preprocessor
+        // make sure we have enough space in buffer
+        ulas_strensr(dst, write + 2);
+        // escape char tokens
         dst->buf[write++] = line[i++];
-      }
-      goto tokdone;
-    case '=':
-    case '<':
-    case '!':
-    case '>':
-      if (line[i + 1] == '=') {
+        if (line[i] && !isspace(line[i])) {
+          dst->buf[write++] = line[i++];
+        }
+        goto tokdone;
+      case '=':
+      case '<':
+      case '!':
+      case '>':
+        if (line[i + 1] == '=') {
+          dst->buf[write] = line[i];
+          i++;
+          write++;
+        }
         dst->buf[write] = line[i];
+        write++;
         i++;
+        goto tokdone;
+      default:
+        if (isspace(line[i])) {
+          goto tokdone;
+        }
+        dst->buf[write] = line[i];
         write++;
+        break;
       }
-      dst->buf[write] = line[i];
-      write++;
       i++;
-      goto tokdone;
-    default:
-      if (isspace(line[i])) {
-        goto tokdone;
-      }
-      dst->buf[write] = line[i];
-      write++;
-      break;
     }
-    i++;
   }
 tokdone:
-  
+
   dst->buf[write] = '\0';
 
   *out_line += i;