Skip to content

Commit

Permalink
Fix issue with tokenizer not correctly handling escape sequences
Browse files Browse the repository at this point in the history
  • Loading branch information
ms-is-coding committed Apr 29, 2023
1 parent 1cbe41e commit 4c9f6b6
Show file tree
Hide file tree
Showing 4 changed files with 13 additions and 6 deletions.
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
out/
tests/
.data/
4 changes: 2 additions & 2 deletions makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ gcc:
$(COMPILE)
run:
$(COMPILE)
./out/debug.bin tests/index.hadron
./out/debug.bin .data/index.hadron
debug:
$(COMPILE)
gdb -ex=r\ tests/index.hadron -ex=set\ confirm\ off -ex=q --silent ./out/debug.bin
gdb -ex=r\ .data/index.hadron -ex=set\ confirm\ off -ex=q --silent ./out/debug.bin
clean:
rm ./hadron
7 changes: 4 additions & 3 deletions src/tokenizer.c
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ Result *tokenize(string code, string fname) {
case ']': pushArray(tokenizer.stack, cloneToken(addToken($BRACKET))); continue;
case '{': pushArray(tokenizer.stack, cloneToken(addToken(CURLY))); continue;
case '}': pushArray(tokenizer.stack, cloneToken(addToken($CURLY))); continue;
case '\n': continue;
case '\n': continue;
case ' ': case '\t': continue;
case '+': {
if (match('=')) addToken(ADD_EQ);
Expand Down Expand Up @@ -149,7 +149,7 @@ Result *tokenize(string code, string fname) {
bool err = false;
while (peekNext() != '"') {
if (peekNext() == '\n') { err = true; break; }
if (current() == '\\') next();
if (peekNext() == '\\') next();
next();
}; if (err) pushArray(tokenizer.errors, error(
"Syntax", fname, "unterminated string", addToken(STR)
Expand Down Expand Up @@ -224,7 +224,8 @@ Result *tokenize(string code, string fname) {
trimArray(tokenizer.tokens);
result->errors = tokenizer.errors;
result->data = tokenizer.tokens;
return result;
printTokens(tokenizer.code, tokenizer.tokens);
return result;
}

void runDistanceCheck(Array *r) {
Expand Down
7 changes: 7 additions & 0 deletions tests/main.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#include <stdio.h>

int main() {
printf("all tests passed\n");

return 0;
}

0 comments on commit 4c9f6b6

Please sign in to comment.