From 3dddb101436d5ac715666fdf3471ebc3990d6966 Mon Sep 17 00:00:00 2001
From: Lukas Krickl <lukas@krickl.dev>
Date: Sun, 19 Nov 2023 09:06:55 +0100
Subject: [PATCH] WIP: tokenizer adding special tokens

---
 include/ulas.h | 10 +++++++++-
 src/test.c     | 15 +++++++++++++++
 src/ulas.c     | 17 ++++++++++++++++-
 3 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/include/ulas.h b/include/ulas.h
index f6955bc..85bf25e 100644
--- a/include/ulas.h
+++ b/include/ulas.h
@@ -92,7 +92,15 @@ struct ulas_str {
 
 // any token before 256 is just the literal char value
 // primitive data types
-enum ulas_type { ULAS_SYMBOL = 256, ULAS_INT, ULAS_STR };
+enum ulas_type {
+  ULAS_SYMBOL = 256,
+  ULAS_INT,
+  ULAS_STR,
+  ULAS_EQ,
+  ULAS_NEQ,
+  ULAS_GTEQ,
+  ULAS_LTEQ
+};
 
 // data type value
 union ulas_val {
diff --git a/src/test.c b/src/test.c
index 61b83ce..6fc8c51 100644
--- a/src/test.c
+++ b/src/test.c
@@ -178,6 +178,16 @@ void test_preproc(void) {
     assert((expected_rc) == rc);                                               \
   }
 
+
+#define ASSERT_TOTOK(expected_val, expected_rc, token)                  \
+  {                                                                            \
+    int rc = 0;                                                                \
+    struct ulas_tok tok = ulas_totok((token), strlen(token), &rc);             \
+    assert((expected_rc) == rc);                                               \
+    assert(tok.type == (expected_val));                                           \
+    free(tok.val.strv);                                                        \
+  }
+
 void test_totok(void) {
   TESTBEGIN("totok");
 
@@ -210,6 +220,11 @@ void test_totok(void) {
 
   ASSERT_UNEXPECTED_TOTOK(-1, "1symbol123");
 
+  // generic tokens with no value 
+  ASSERT_TOTOK(ULAS_EQ, 0, "==");
+  ASSERT_TOTOK('=', 0, "=");
+  ASSERT_TOTOK('+', 0, "+");
+
   TESTEND("totok");
 }
 
diff --git a/src/ulas.c b/src/ulas.c
index fc63f82..8cd5537 100644
--- a/src/ulas.c
+++ b/src/ulas.c
@@ -267,7 +267,6 @@ struct ulas_tok ulas_totok(char *buf, unsigned long n, int *rc) {
   case '-':
   case '*':
   case '/':
-  case '!':
   case '~':
   case '|':
   case '&':
@@ -310,6 +309,22 @@ struct ulas_tok ulas_totok(char *buf, unsigned long n, int *rc) {
     }
     buf++;
     break;
+  case '=':
+    if (*buf == '=') {
+      tok.type = ULAS_EQ;
+      buf++;
+    } else {
+      tok.type = first;
+    }
+    break;
+  case '!':
+    if (*buf == '=') {
+      tok.type = ULAS_NEQ;
+      buf++;
+    } else {
+      tok.type = first;
+    }
+    break;
   default:
     if (isdigit(first)) {
       // integer
-- 
2.30.2