|
2 | 2 |
|
3 | 3 | std::vector<Token> Lexer::tokenize() {
|
4 | 4 | std::vector<Token> tokens;
|
| 5 | + bool lineHasTokens = false; |
| 6 | + |
5 | 7 | while (pos < source.length()) {
|
6 | 8 | char current = peek();
|
7 | 9 |
|
8 |
| - if (std::isdigit(current) || current == '-') { |
9 |
| - tokens.push_back(readNumber()); |
10 |
| - } else if (std::isalpha(current) || current == '_') { |
11 |
| - tokens.push_back(readIdentifier()); |
12 |
| - } else if (current == '"') { |
13 |
| - tokens.push_back(readString()); |
14 |
| - } else if (current == ',') { |
15 |
| - consume(); |
16 |
| - tokens.push_back({ TokenType::COMMA, ",", line }); |
17 |
| - } else if (current == '\n') { |
| 10 | + if (current == '\n') { |
18 | 11 | consume();
|
19 |
| - tokens.push_back({ TokenType::END_OF_LINE, "EOL", line }); |
| 12 | + if (lineHasTokens) { |
| 13 | + tokens.push_back({ TokenType::END_OF_LINE, "EOL", line }); |
| 14 | + lineHasTokens = false; |
| 15 | + } |
20 | 16 | line++;
|
21 |
| - } else { |
| 17 | + } else if (std::isspace(current)) { |
22 | 18 | consume();
|
| 19 | + } else { |
| 20 | + if (std::isdigit(current) || current == '-') { |
| 21 | + tokens.push_back(readNumber()); |
| 22 | + } else if (std::isalpha(current) || current == '_') { |
| 23 | + tokens.push_back(readIdentifier()); |
| 24 | + } else if (current == '"') { |
| 25 | + tokens.push_back(readString()); |
| 26 | + } else if (current == ',') { |
| 27 | + consume(); |
| 28 | + tokens.push_back({ TokenType::COMMA, ",", line }); |
| 29 | + } else if (current == '\n') { |
| 30 | + consume(); |
| 31 | + tokens.push_back({ TokenType::END_OF_LINE, "EOL", line }); |
| 32 | + line++; |
| 33 | + } else { |
| 34 | + consume(); |
| 35 | + } |
| 36 | + |
| 37 | + lineHasTokens = true; |
23 | 38 | }
|
24 | 39 | }
|
25 | 40 |
|
|
0 commit comments