From: Lukas Krickl <lukas@krickl.dev>
Date: Tue, 17 Mar 2026 06:28:43 +0000 (+0100)
Subject: script: tokenizer now checks for term chars when a token ends
X-Git-Url: https://git.krickl.dev/?a=commitdiff_plain;h=bb77401d11df31b5aecc691c99e2a47adb69dc79;p=lrts%2F.git

script: tokenizer now checks for term chars when a token ends
---

diff --git a/src/l_lsl.c b/src/l_lsl.c
index 8c47b6c..5efae9c 100644
--- a/src/l_lsl.c
+++ b/src/l_lsl.c
@@ -18,6 +18,7 @@ int l_lsl_err(struct l_lsl_vm *v, enum l_lsl_error err, const char *fmt, ...) {
 	va_list args;
 
 	va_start(args, fmt);
+	u_fprintf(u_stderr, "error at %s:%d: ", v->path, v->line_num);
 	res = u_vfprintf(u_stderr, fmt, args);
 	va_end(args);
 
@@ -32,6 +33,7 @@ int l_lsl_warn(struct l_lsl_vm *v, enum l_lsl_error err, const char *fmt, ...) {
 	LRTS_UNUSED(err);
 
 	va_start(args, fmt);
+	u_fprintf(u_stderr, "warning at %s:%d: ", v->path, v->line_num);
 	res = u_vfprintf(u_stderr, fmt, args);
 	va_end(args);
 	return res;
@@ -70,6 +72,15 @@ lrts_bool l_lsl_isident(char c) {
 		|| c == '_' || l_lsl_isnum(c);
 }
 
+lrts_bool l_lsl_isspace(char c) {
+	return c == ' ' || c == '\t'
+		|| c == '\n' || c == '\r';
+}
+
+lrts_bool l_lsl_isterm(char c) {
+	return c == '\0' || l_lsl_isspace(c);
+}
+
 /* peeks current token */
 #define l_lsl_lex_peek(v, code) \
 	(code)[v->lex_pos]
@@ -97,8 +108,17 @@ struct l_lsl_token l_lsl_next_token(struct l_lsl_vm *v, const char *code) {
 	u_memset(&t, 0, sizeof(t));
 	t.val = tokbuf;
 
+
 	
 	c = l_lsl_lex_peek(v, code);
+	/* consume all spaces */
+	while (l_lsl_isspace(c)) {
+		if (c == '\n') {
+			v->line_num++;
+		}
+		c = l_lsl_lex_advance(v, code);
+	}
+
 	if (l_lsl_isnum(c)) {
 		t.type = L_LSL_TOK_INT;
 		if (c == '0') {
@@ -153,6 +173,13 @@ struct l_lsl_token l_lsl_next_token(struct l_lsl_vm *v, const char *code) {
 				
 	l_lsl_lex_term_token(t);
 
+	/* end of a token must be a term */
+	c = l_lsl_lex_peek(v, code);
+	if (!l_lsl_isterm(c)) {
+		l_lsl_err(v, L_LSL_ERR_LEX, "unexpected end of token\n", c);
+		t.type = L_LSL_TOK_NONE;
+	}
+
 	return t;
 }
 
@@ -204,9 +231,11 @@ struct l_lsl_value* l_lsl_compile_file(struct l_lsl_vm *v, const char *path) {
 
 	u32 prev_lex_pos = v->lex_pos;
 	const char *prev_file_name = v->path;
+	u32 prev_line_num = v->line_num;
 
 	v->lex_pos = 0;
 	v->path = path;
+	v->line_num = 1;
 
 	if (src == LRTS_NULL) {
 		u_log(U_LOG_CRIT, "%s: No such file or directory\n", path);
@@ -220,6 +249,7 @@ struct l_lsl_value* l_lsl_compile_file(struct l_lsl_vm *v, const char *path) {
 
 	v->lex_pos = prev_lex_pos;
 	v->path = prev_file_name;
+	v->line_num = prev_line_num;
 	return val;
 }
 
diff --git a/src/l_lsl.h b/src/l_lsl.h
index 2ad3f76..861b530 100644
--- a/src/l_lsl.h
+++ b/src/l_lsl.h
@@ -91,6 +91,7 @@ struct l_lsl_vm {
 
 	const char *path;
 	u32 lex_pos;
+	u32 line_num;
 
 	struct l_lsl_gc gc;