WIP: tokenizer adding special tokens

diff --git a/include/ulas.h b/include/ulas.h
index f6955bcef726e9ad30426924b5fff030999fc048..85bf25eb417f3354dc662e01794c39f61c07bc7d 100644 (file)
--- a/include/ulas.h
+++ b/include/ulas.h
@@ -92,7 +92,15 @@ struct ulas_str {
 
 // any token before 256 is just the literal char value
 // primitive data types
-enum ulas_type { ULAS_SYMBOL = 256, ULAS_INT, ULAS_STR };
+enum ulas_type {
+  ULAS_SYMBOL = 256,
+  ULAS_INT,
+  ULAS_STR,
+  ULAS_EQ,
+  ULAS_NEQ,
+  ULAS_GTEQ,
+  ULAS_LTEQ
+};
 
 // data type value
 union ulas_val {

diff --git a/src/test.c b/src/test.c
index 61b83ceac7bc29af2908574f0426a48fbc4b1119..6fc8c51f1cbfcc8d48560f9820cf90c329710a73 100644 (file)
--- a/src/test.c
+++ b/src/test.c
@@ -178,6 +178,16 @@ void test_preproc(void) {
     assert((expected_rc) == rc);                                               \
   }
 
+
+#define ASSERT_TOTOK(expected_val, expected_rc, token)                  \
+  {                                                                            \
+    int rc = 0;                                                                \
+    struct ulas_tok tok = ulas_totok((token), strlen(token), &rc);             \
+    assert((expected_rc) == rc);                                               \
+    assert(tok.type == (expected_val));                                           \
+    free(tok.val.strv);                                                        \
+  }
+
 void test_totok(void) {
   TESTBEGIN("totok");
 
@@ -210,6 +220,11 @@ void test_totok(void) {
 
   ASSERT_UNEXPECTED_TOTOK(-1, "1symbol123");
 
+  // generic tokens with no value 
+  ASSERT_TOTOK(ULAS_EQ, 0, "==");
+  ASSERT_TOTOK('=', 0, "=");
+  ASSERT_TOTOK('+', 0, "+");
+
   TESTEND("totok");
 }
 

diff --git a/src/ulas.c b/src/ulas.c
index fc63f826bbeda68900092acad6e087e99614e1af..8cd55378ae45868770021d82f054e607b20734a2 100644 (file)
--- a/src/ulas.c
+++ b/src/ulas.c
@@ -267,7 +267,6 @@ struct ulas_tok ulas_totok(char *buf, unsigned long n, int *rc) {
   case '-':
   case '*':
   case '/':
-  case '!':
   case '~':
   case '|':
   case '&':
@@ -310,6 +309,22 @@ struct ulas_tok ulas_totok(char *buf, unsigned long n, int *rc) {
     }
     buf++;
     break;
+  case '=':
+    if (*buf == '=') {
+      tok.type = ULAS_EQ;
+      buf++;
+    } else {
+      tok.type = first;
+    }
+    break;
+  case '!':
+    if (*buf == '=') {
+      tok.type = ULAS_NEQ;
+      buf++;
+    } else {
+      tok.type = first;
+    }
+    break;
   default:
     if (isdigit(first)) {
       // integer