From 5b85ff8864a6606edda03d36956e0efc788f0cf4 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Tue, 9 Jul 2019 23:15:06 +0200 Subject: [PATCH] fixed a lot of D completion edge cases --- dub.json | 2 +- source/app.d | 21 +++--- source/dietc/complete.d | 78 +++++++++++++++++++- source/dietc/lexer.d | 31 +++++++- source/dietc/parser.d | 154 ++++++++++++++++++++++++++++++++++++---- 5 files changed, 256 insertions(+), 30 deletions(-) diff --git a/dub.json b/dub.json index 59b0775..9b537ce 100644 --- a/dub.json +++ b/dub.json @@ -2,5 +2,5 @@ "name": "diet-complete", "description": "Error correcting diet parser with integrated auto-completion and neat AST tools", "license": "MIT", - "copyright": "Copyright © 2018 webfreak" + "copyright": "Copyright © 2018-2019 webfreak" } \ No newline at end of file diff --git a/source/app.d b/source/app.d index f041513..c6f91d8 100644 --- a/source/app.d +++ b/source/app.d @@ -16,12 +16,9 @@ void main(string[] args) DietInput input; input.file = "stdin"; input.code = q{doctype html -html(lang="de") - head - title Hello World - body - - foreach (index; 0 .. 10) - ul= i}; +html + +}; auto parser = new ASTParser; parser.input = input.save; @@ -49,11 +46,11 @@ html(lang="de") writeln(); } - testComplete(24); + testComplete(19); - string code; - size_t offset; - complete.extractD(98, code, offset); - writeln("D:\n", code); - writeln(offset == size_t.max ? "not found" : offset.to!string); + // string code; + // size_t offset; + // complete.extractD(98, code, offset); + // writeln("D:\n", code); + // writeln(offset == size_t.max ? "not found" : offset.to!string); } diff --git a/source/dietc/complete.d b/source/dietc/complete.d index 21525c2..9146e1c 100644 --- a/source/dietc/complete.d +++ b/source/dietc/complete.d @@ -446,9 +446,16 @@ void extractD(AST root, size_t offset, out string code, out size_t codeOffset) static foreach (T; AliasSeq!(Expression, Assignment, RawAssignment)) override void visit(T expr) in (expr !is null) { + static if (is(T == Expression)) + enum typeOffset = 0; + else static if (is(T == Assignment)) + enum typeOffset = 1; // offset the equal of tag= + else static if (is(T == RawAssignment)) + enum typeOffset = 2; // offset the bang-equal of tag!= + code ~= "__diet_value("; if (offset.withinRange(expr.token.range)) - codeOffset = code.length + (offset - expr.token.range[0] - expr.token.content.length); + codeOffset = code.length + (offset - typeOffset - expr.token.range[0]); code ~= expr.content; code ~= ");"; } @@ -526,3 +533,72 @@ unittest assert(testComplete("", 0).canFind!(a => a.text == "textarea")); assert(testComplete("div\n\t", 5).canFind!(a => a.text == "textarea")); } + +unittest +{ + import std.conv; + + DietInput input; + input.file = "stdin"; + input.code = `foo + - int item = 3; + p #{item.foobar} bar + a(attr=foo.bar)= foo.bar +`; + auto c = new DietComplete(input, cast(FileProvider)(name) { + assert(false, "Can't import " ~ name ~ " in test"); + }); + + assert(c.completeAt(23).canFind!(a => a.text == "pre")); + assert(c.completeAt(39).length == 0); + assert(c.completeAt(38).length == 0); + assert(c.completeAt(24).length == 0); + assert(c.completeAt(25).length == 0); + + void checkDBounds(size_t start, size_t end) + { + assert(c.completeAt(start - 1) !is Completion.completeD); + assert(c.completeAt(start) is Completion.completeD); + assert(c.completeAt(end) is Completion.completeD); + assert(c.completeAt(end + 1) !is Completion.completeD); + } + + checkDBounds(6, 20); + checkDBounds(26, 37); + checkDBounds(51, 58); + checkDBounds(60, 68); + + string code; + size_t offset; + + c.extractD(28, code, offset); + assert(code == `void __diet_document() {{ int item = 3;{__diet_value(item.foobar);}{{__diet_value(foo.bar);}__diet_value( foo.bar);}}}`); + assert(offset == 55); + + c.extractD(37, code, offset); + assert(offset == 64); + + c.extractD(38, code, offset); + assert(offset == size_t.max); + + c.extractD(25, code, offset); + assert(offset == size_t.max); + + c.extractD(7, code, offset); + assert(offset == 26); + + c.extractD(20, code, offset); + assert(offset == 39); + + c.extractD(51, code, offset); + assert(offset == 82); + + c.extractD(58, code, offset); + assert(offset == 89); + + c.extractD(61, code, offset); + assert(offset == 106); + + c.extractD(68, code, offset); + assert(offset == 113); +} diff --git a/source/dietc/lexer.d b/source/dietc/lexer.d index 45b339d..e923512 100644 --- a/source/dietc/lexer.d +++ b/source/dietc/lexer.d @@ -224,8 +224,8 @@ struct DietInput Token tok = front; auto ret = this.match(type, match); if (!ret) - errors.expect(this, at, (match.length ? "'" ~ match ~ "'" - : type.to!string) ~ ", but got " ~ tok.to!string, srcfile, srcline); + errors.expect(this, at, (match.length + ? "'" ~ match ~ "'" : type.to!string) ~ ", but got " ~ tok.to!string, srcfile, srcline); return ret; } @@ -400,7 +400,9 @@ struct DietInput return ret; } else - return [Token(TokenType.whitespace, code[start .. index], [start, index])]; + return [ + Token(TokenType.whitespace, code[start .. index], [start, index]) + ]; } else if (c.tagIdentifierValidator) { @@ -489,3 +491,26 @@ string indent(string s, string indentation = "\t") .map!(a => a.strip.length ? indentation ~ a : a) .join(""); } + +unittest +{ + import std.array : array; + + DietInput input; + input.file = "stdin"; + input.code = q{doctype html +html + +}; + + auto tokens = input.array; + assert(input.errors.length == 0); + + with (TokenType) + assert(tokens == [ + Token(identifier, "doctype", [0, 7]), Token(whitespace, " ", [7, 8]), + Token(identifier, "html", [8, 12]), Token(newline, "\n", [12, 13]), + Token(identifier, "html", [13, 17]), Token(newline, "\n", [17, 18]), + Token(newline, "\n", [18, 19]) + ]); +} diff --git a/source/dietc/parser.d b/source/dietc/parser.d index 1733cef..0d3f421 100644 --- a/source/dietc/parser.d +++ b/source/dietc/parser.d @@ -3,11 +3,13 @@ module dietc.parser; import dietc.lexer; import std.algorithm; -import std.conv : to, text; +import std.array : join; +import std.conv : text, to; import std.meta : AliasSeq; -alias ASTClasses = AliasSeq!(Document, HiddenComment, Comment, DStatement, DietFilter, TagNode, TagNode.AttributeAST, - RawAssignment, Assignment, StringTagContents, TextLine, XMLNode, PipeText, Expression, TextLine.PartAST); +alias ASTClasses = AliasSeq!(Document, HiddenComment, Comment, DStatement, + DietFilter, TagNode, TagNode.AttributeAST, RawAssignment, + Assignment, StringTagContents, TextLine, XMLNode, PipeText, Expression, TextLine.PartAST); interface AST { @@ -18,12 +20,14 @@ interface AST abstract class ASTVisitor { static foreach (T; ASTClasses) - void visit(T ast) in (ast !is null) - { - ast.accept(this); - } + void visit(T ast) + in(ast !is null) + { + ast.accept(this); + } - void visit(AST ast) in (ast !is null) + void visit(AST ast) + in(ast !is null) { static foreach (T; ASTClasses) if (cast(T) ast) @@ -182,7 +186,7 @@ abstract class StringNode : Node, IStringContainer { import std.array : join; - string ret = text('(', (cast(Object)this).classinfo.name, `) "`, content, '"'); + string ret = text('(', (cast(Object) this).classinfo.name, `) "`, content, '"'); foreach (child; children) ret ~= "\n" ~ child.to!string.indent; return ret; @@ -425,7 +429,7 @@ class StringTagContents : TagContents, IStringContainer { import std.array : join; - string ret = text('(', (cast(Object)this).classinfo.name, `) "`, content, '"'); + string ret = text('(', (cast(Object) this).classinfo.name, `) "`, content, '"'); return ret; } } @@ -1142,6 +1146,7 @@ struct ASTParser input.errors.expect(input, input.front.range[0], "'" ~ (cast(char[])(cast(ubyte[]) stack).retro.chain.array).idup ~ "' before ')'"); tok.range[1] = input.front.range[0]; + tok.content = input.code[tok.range[0] .. tok.range[1]]; return new Expression(tok, ret); } @@ -1150,7 +1155,6 @@ struct ASTParser auto save = input.save; auto tok = input.front; - input.popFront; Token[] classes, ids; @@ -1187,10 +1191,11 @@ struct ASTParser Token tag; bool match; - if (input.front.content == "." || input.front.content == "#") + if (tok.content == "." || tok.content == "#") { tag = tok; tag.range[1] = tag.range[0]; + tag.type = TokenType.identifier; tag.content = "div"; while (parseClassOrID()) { @@ -1199,6 +1204,7 @@ struct ASTParser } else if (tok.type == TokenType.identifier) { + input.popFront; tag = tok; if (!tok.content.validateTagIdentifier) input.errors.expect(input, tok.range[0], "identifier of type [-:_0-9a-zA-Z]+"); @@ -1420,7 +1426,7 @@ struct ASTParser /// inclusiveEnd = true if an AST [1 .. 3] should be matched for index 3. /// Returns: A path of AST nodes starting at the broadest object (Document) down to the finest object. AST[] searchAST(size_t offset, bool inclusiveStart = true, bool inclusiveEnd = true) - out (r; r.length > 0) + out(r; r.length > 0) { AST[] ret = [root]; @@ -1457,3 +1463,125 @@ void skipAllWhiteGetDetent(ref DietInput input, ref bool detented) if (c[1]) detented = true; } + +private void assertToken(Token token, TokenType type, string content) +{ + assert(token.type == type); + assert(token.content == content); +} + +private void assertToken(Token token, TokenType type, string content, size_t[2] range) +{ + assert(token.type == type); + assert(token.content == content); + assert(token.range == range); +} + +unittest +{ + DietInput input; + input.file = "stdin"; + input.code = q{doctype html +html + +}; + + auto parser = new ASTParser; + parser.input = input.save; + parser.parseDocument(); + + assert(parser.input.errors.length == 0); + + assert(parser.root); + assert(parser.root.token.range == [0, 17]); + assert(parser.root.children.length == 2); + + auto doctype = cast(TagNode) parser.root.children[0]; + auto html = cast(TagNode) parser.root.children[1]; + + assert(doctype); + assert(html); + + doctype.tag.assertToken(TokenType.identifier, "doctype"); + html.tag.assertToken(TokenType.identifier, "html"); +} + +unittest +{ + DietInput input; + input.file = "stdin"; + input.code = `foo + .bar1 text1 + .bar2 text2 +`; + + auto parser = new ASTParser; + parser.input = input.save; + parser.parseDocument(); + + assert(parser.input.errors.length == 0); + + assert(parser.root); + assert(parser.root.children.length == 1); + auto root = cast(TagNode) parser.root.children[0]; + assert(root); + root.tag.assertToken(TokenType.identifier, "foo"); + assert(root.children.length == 2); + + auto bar1 = cast(TagNode) root.children[0]; + auto bar2 = cast(TagNode) root.children[1]; + assert(bar1); + assert(bar2); + + bar1.tag.assertToken(TokenType.identifier, "div", [5, 5]); + bar2.tag.assertToken(TokenType.identifier, "div", [18, 18]); + + assert(cast(TextLine) bar1.contents, "Expected string contents but got " ~ bar1.contents.to!string); + assert(cast(TextLine) bar2.contents, "Expected string contents but got " ~ bar2.contents.to!string); + + assert((cast(TextLine) bar1.contents)._parts.length == 1); + assert((cast(TextLine) bar2.contents)._parts.length == 1); + assert((cast(TextLine) bar1.contents)._parts[0].raw == "text1"); + assert((cast(TextLine) bar2.contents)._parts[0].raw == "text2"); +} + +unittest +{ + DietInput input; + input.file = "stdin"; + input.code = `foo + - int item = 3; + p #{item.foo} bar +`; + + auto parser = new ASTParser; + parser.input = input.save; + parser.parseDocument(); + + assert(parser.input.errors.length == 0); + + assert(parser.root); + assert(parser.root.children.length == 1); + auto root = cast(TagNode) parser.root.children[0]; + assert(root); + root.tag.assertToken(TokenType.identifier, "foo"); + assert(root.children.length == 2); + + auto code = cast(DStatement) root.children[0]; + auto paragraph = cast(TagNode) root.children[1]; + assert(code); + assert(paragraph); + + assert(code.content == " int item = 3;"); + paragraph.tag.assertToken(TokenType.identifier, "p"); + + auto content = cast(TextLine) paragraph.contents; + assert(content); + assert(content._parts.length == 2); + assert(content._parts[0].inlineExpr); + assert(content._parts[0].inlineExpr.token.range[0] == 26); + assert(content._parts[0].inlineExpr.token.range[1] == 34); + assert(content._parts[0].inlineExpr.token.content == "item.foo"); + assert(content._parts[0].inlineExpr.content == "item.foo"); + assert(content._parts[1].raw == " bar"); +}