Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix tokenizer error when empty directive is used #1763

Merged
merged 1 commit into from
Jan 31, 2024
Merged

Fix tokenizer error when empty directive is used #1763

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions src/Framework/Framework/Compilation/Parser/Dothtml/Tokenizer/DothtmlTokenizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,22 +145,22 @@ private void ReadDirective()
}
SkipWhitespace(false);

if (Peek() is '\r' or '\n' or NullChar)
{
// empty value
CreateToken(DothtmlTokenType.DirectiveValue);
SkipWhitespace();
return;
}

// whitespace
if (LastToken!.Type != DothtmlTokenType.WhiteSpace)
{
CreateToken(DothtmlTokenType.WhiteSpace, errorProvider: t => CreateTokenError(t, DothtmlTokenType.DirectiveStart, DothtmlTokenizerErrors.DirectiveValueExpected));
}

// directive value
if (Peek() == '\r' || Peek() == '\n' || Peek() == NullChar)
{
CreateToken(DothtmlTokenType.DirectiveValue, errorProvider: t => CreateTokenError(t, DothtmlTokenType.DirectiveStart, DothtmlTokenizerErrors.DirectiveValueExpected));
SkipWhitespace();
}
else
{
ReadTextUntilNewLine(DothtmlTokenType.DirectiveValue);
}
ReadTextUntilNewLine(DothtmlTokenType.DirectiveValue);
}

/// <summary>
Expand Down
51 changes: 30 additions & 21 deletions src/Tests/Parser/Dothtml/DothtmlTokenizerDirectivesTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,19 +95,15 @@ public void DothtmlTokenizer_DirectiveParsing_Invalid_OnlyAtSymbol()
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.DirectiveName, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.WhiteSpace, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.IsFalse(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.DirectiveValue, tokenizer.Tokens[i++].Type);
}

[TestMethod]
public void DothtmlTokenizer_DirectiveParsing_Invalid_AtSymbol_DirectiveName()
public void DothtmlTokenizer_DirectiveParsing_Invalid_ValueWithoutSpace()
{
var input = @"@viewmodel";
var input = @"@viewmodel=something";

// parse
var tokenizer = new DothtmlTokenizer();
Expand All @@ -124,7 +120,28 @@ public void DothtmlTokenizer_DirectiveParsing_Invalid_AtSymbol_DirectiveName()
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.WhiteSpace, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.AreEqual("=something", tokenizer.Tokens[i].Text);
Assert.IsFalse(tokenizer.Tokens[i].HasError);
Assert.AreEqual(DothtmlTokenType.DirectiveValue, tokenizer.Tokens[i++].Type);
}

[TestMethod]
public void DothtmlTokenizer_DirectiveParsing_Valid_EmptyDirective()
{
var input = @"@viewmodel";

// parse
var tokenizer = new DothtmlTokenizer();
tokenizer.Tokenize(input);
CheckForErrors(tokenizer, input.Length);

var i = 0;
Assert.AreEqual(DothtmlTokenType.DirectiveStart, tokenizer.Tokens[i++].Type);

Assert.AreEqual("viewmodel", tokenizer.Tokens[i].Text);
Assert.AreEqual(DothtmlTokenType.DirectiveName, tokenizer.Tokens[i++].Type);

Assert.IsFalse(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.DirectiveValue, tokenizer.Tokens[i++].Type);
}
Expand Down Expand Up @@ -170,11 +187,7 @@ public void DothtmlTokenizer_DirectiveParsing_Invalid_AtSymbol_NewLine_Content()
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.DirectiveName, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.WhiteSpace, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.IsFalse(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.DirectiveValue, tokenizer.Tokens[i++].Type);

Expand All @@ -184,7 +197,7 @@ public void DothtmlTokenizer_DirectiveParsing_Invalid_AtSymbol_NewLine_Content()
}

[TestMethod]
public void DothtmlTokenizer_DirectiveParsing_Invalid_AtSymbol_DirectiveName_NewLine_Content()
public void DothtmlTokenizer_DirectiveParsing_DirectiveName_EmptyValue_NewLine_Content()
{
var input = "@viewmodel\ntest";

Expand All @@ -199,11 +212,7 @@ public void DothtmlTokenizer_DirectiveParsing_Invalid_AtSymbol_DirectiveName_New
Assert.AreEqual("viewmodel", tokenizer.Tokens[i].Text);
Assert.AreEqual(DothtmlTokenType.DirectiveName, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.WhiteSpace, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.IsFalse(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.DirectiveValue, tokenizer.Tokens[i++].Type);

Expand Down Expand Up @@ -233,7 +242,7 @@ public void DothtmlTokenizer_DirectiveParsing_Invalid_AtSymbol_DirectiveName_Spa
Assert.AreEqual(2, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.WhiteSpace, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.IsFalse(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.DirectiveValue, tokenizer.Tokens[i++].Type);

Expand Down Expand Up @@ -265,7 +274,7 @@ public void DothtmlTokenizer_DirectiveParsing_Invalid_AtSymbol_Space_NewLine_Con
Assert.AreEqual(2, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.WhiteSpace, tokenizer.Tokens[i++].Type);

Assert.IsTrue(tokenizer.Tokens[i].HasError);
Assert.IsFalse(tokenizer.Tokens[i].HasError);
Assert.AreEqual(0, tokenizer.Tokens[i].Length);
Assert.AreEqual(DothtmlTokenType.DirectiveValue, tokenizer.Tokens[i++].Type);

Expand Down
10 changes: 10 additions & 0 deletions src/Tests/Runtime/ControlTree/DefaultControlTreeResolver/ViewModelDirectiveTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,16 @@ public void ResolvedTree_UnknownViewModelType()
Assert.IsTrue(directiveNode.NodeErrors.First().Contains("Could not resolve type"));
}

[TestMethod]
public void ResolvedTree_EmptyViewModelType()
{
var root = ParseSource(@"@viewModel
");

var directiveNode = ((DothtmlRootNode)root.DothtmlNode).Directives.First();
Assert.IsTrue(directiveNode.HasNodeErrors);
}

[TestMethod]
public void ResolvedTree_ViewModel_GenericType()
{
Expand Down
Loading