lexer: recognize identifiers, comma, and equals tokens
- Add identifier scanning: starts with letter/underscore, continues with
letters/digits/underscores. Produces Ident tokens.
- Add comma and equals to single-char token switch.
- Add isLetter() helper.
- Add 9 new tests covering: single ident, multi-char ident, ident with
digits, ident with underscore, comma, equals, function definition
syntax, function call with args, multi-param func def, func call in
expression.
diff --git a/lexer/lexer.go b/lexer/lexer.go
index 0914b72..d1f55d5 100644
--- a/lexer/lexer.go
+++ b/lexer/lexer.go
@@ -41,6 +41,20 @@
continue
}
+ // Identifier: starts with letter, continues with letters/digits.
+ if isLetter(ch) {
+ start := i
+ for i < len(input) && (isLetter(input[i]) || isDigit(input[i])) {
+ i++
+ }
+ tokens = append(tokens, token.Token{
+ Type: token.Ident,
+ Literal: input[start:i],
+ Pos: start,
+ })
+ continue
+ }
+
// Single-character tokens.
var typ token.Type
switch ch {
@@ -56,6 +70,10 @@
typ = token.LParen
case ')':
typ = token.RParen
+ case ',':
+ typ = token.Comma
+ case '=':
+ typ = token.Equals
default:
return nil, fmt.Errorf("unexpected character %q at position %d", string(ch), i)
}
@@ -81,3 +99,7 @@
func isDigit(ch byte) bool {
return ch >= '0' && ch <= '9'
}
+
+func isLetter(ch byte) bool {
+ return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_'
+}