script: tokenizer now checks for term chars when a token ends

diff --git a/src/l_lsl.c b/src/l_lsl.c
index 8c47b6c3f5b0c08bac8ba2929e20d0107edf651a..5efae9c899e53401bdb0c12c6cbb5cdb11c88874 100644 (file)
--- a/src/l_lsl.c
+++ b/src/l_lsl.c
@@ -18,6 +18,7 @@ int l_lsl_err(struct l_lsl_vm *v, enum l_lsl_error err, const char *fmt, ...) {
        va_list args;
 
        va_start(args, fmt);
+       u_fprintf(u_stderr, "error at %s:%d: ", v->path, v->line_num);
        res = u_vfprintf(u_stderr, fmt, args);
        va_end(args);
 
@@ -32,6 +33,7 @@ int l_lsl_warn(struct l_lsl_vm *v, enum l_lsl_error err, const char *fmt, ...) {
        LRTS_UNUSED(err);
 
        va_start(args, fmt);
+       u_fprintf(u_stderr, "warning at %s:%d: ", v->path, v->line_num);
        res = u_vfprintf(u_stderr, fmt, args);
        va_end(args);
        return res;
@@ -70,6 +72,15 @@ lrts_bool l_lsl_isident(char c) {
                || c == '_' || l_lsl_isnum(c);
 }
 
+lrts_bool l_lsl_isspace(char c) {
+       return c == ' ' || c == '\t'
+               || c == '\n' || c == '\r';
+}
+
+lrts_bool l_lsl_isterm(char c) {
+       return c == '\0' || l_lsl_isspace(c);
+}
+
 /* peeks current token */
 #define l_lsl_lex_peek(v, code) \
        (code)[v->lex_pos]
@@ -97,8 +108,17 @@ struct l_lsl_token l_lsl_next_token(struct l_lsl_vm *v, const char *code) {
        u_memset(&t, 0, sizeof(t));
        t.val = tokbuf;
 
+
        
        c = l_lsl_lex_peek(v, code);
+       /* consume all spaces */
+       while (l_lsl_isspace(c)) {
+               if (c == '\n') {
+                       v->line_num++;
+               }
+               c = l_lsl_lex_advance(v, code);
+       }
+
        if (l_lsl_isnum(c)) {
                t.type = L_LSL_TOK_INT;
                if (c == '0') {
@@ -153,6 +173,13 @@ struct l_lsl_token l_lsl_next_token(struct l_lsl_vm *v, const char *code) {
                                
        l_lsl_lex_term_token(t);
 
+       /* end of a token must be a term */
+       c = l_lsl_lex_peek(v, code);
+       if (!l_lsl_isterm(c)) {
+               l_lsl_err(v, L_LSL_ERR_LEX, "unexpected end of token\n", c);
+               t.type = L_LSL_TOK_NONE;
+       }
+
        return t;
 }
 
@@ -204,9 +231,11 @@ struct l_lsl_value* l_lsl_compile_file(struct l_lsl_vm *v, const char *path) {
 
        u32 prev_lex_pos = v->lex_pos;
        const char *prev_file_name = v->path;
+       u32 prev_line_num = v->line_num;
 
        v->lex_pos = 0;
        v->path = path;
+       v->line_num = 1;
 
        if (src == LRTS_NULL) {
                u_log(U_LOG_CRIT, "%s: No such file or directory\n", path);
@@ -220,6 +249,7 @@ struct l_lsl_value* l_lsl_compile_file(struct l_lsl_vm *v, const char *path) {
 
        v->lex_pos = prev_lex_pos;
        v->path = prev_file_name;
+       v->line_num = prev_line_num;
        return val;
 }
 

diff --git a/src/l_lsl.h b/src/l_lsl.h
index 2ad3f761d95a9fd5265943053eafb925b742cb8f..861b5309b38a671fe6a0ed72087383fa3bf9b7ac 100644 (file)
--- a/src/l_lsl.h
+++ b/src/l_lsl.h
@@ -91,6 +91,7 @@ struct l_lsl_vm {
 
        const char *path;
        u32 lex_pos;
+       u32 line_num;
 
        struct l_lsl_gc gc;