include documented unittests inside documentation

dsymbol/src/dsymbol/conversion/first.d

@@ -83,6 +83,11 @@ final class FirstPass : ASTVisitor
 
 	override void visit(const Unittest u)
 	{
+		if (previousSymbol && previousSymbol.acSymbol)
+			makeExampleDocumentation(u, previousSymbol.acSymbol.doc);
+
+		auto associated = previousSymbol;
+		scope(exit) previousSymbol = associated;
 		// Create a dummy symbol because we don't want unit test symbols leaking
 		// into the symbol they're declared in.
 		pushSymbol(UNITTEST_SYMBOL_NAME,
@@ -128,7 +133,7 @@ final class FirstPass : ASTVisitor
 				dec.name.index, dec.returnType);
 		scope (exit) popSymbol();
 		currentSymbol.acSymbol.protection = protection.current;
-		currentSymbol.acSymbol.doc = makeDocumentation(dec.comment);
+		makeDocumentation(currentSymbol.acSymbol.doc, dec.comment);
 
 		istring lastComment = this.lastComment;
 		this.lastComment = istring.init;
@@ -244,7 +249,7 @@ final class FirstPass : ASTVisitor
 				addTypeToLookups(symbol.typeLookups, dec.type);
 			symbol.parent = currentSymbol;
 			symbol.acSymbol.protection = protection.current;
-			symbol.acSymbol.doc = makeDocumentation(declarator.comment);
+			makeDocumentation(symbol.acSymbol.doc, declarator.comment);
 			currentSymbol.addChild(symbol, true);
 			currentScope.addSymbol(symbol.acSymbol, false);
 
@@ -266,7 +271,7 @@ final class FirstPass : ASTVisitor
 				symbol.parent = currentSymbol;
 				populateInitializer(symbol, part.initializer);
 				symbol.acSymbol.protection = protection.current;
-				symbol.acSymbol.doc = makeDocumentation(dec.comment);
+				makeDocumentation(symbol.acSymbol.doc, dec.comment);
 				currentSymbol.addChild(symbol, true);
 				currentScope.addSymbol(symbol.acSymbol, false);
 
@@ -295,7 +300,7 @@ final class FirstPass : ASTVisitor
 				currentSymbol.addChild(symbol, true);
 				currentScope.addSymbol(symbol.acSymbol, false);
 				symbol.acSymbol.protection = protection.current;
-				symbol.acSymbol.doc = makeDocumentation(aliasDeclaration.comment);
+				makeDocumentation(symbol.acSymbol.doc, aliasDeclaration.comment);
 			}
 		}
 		else
@@ -311,7 +316,7 @@ final class FirstPass : ASTVisitor
 				currentSymbol.addChild(symbol, true);
 				currentScope.addSymbol(symbol.acSymbol, false);
 				symbol.acSymbol.protection = protection.current;
-				symbol.acSymbol.doc = makeDocumentation(aliasDeclaration.comment);
+				makeDocumentation(symbol.acSymbol.doc, aliasDeclaration.comment);
 			}
 		}
 	}
@@ -393,7 +398,7 @@ final class FirstPass : ASTVisitor
 		symbol.parent = currentSymbol;
 		currentSymbol.addChild(symbol, true);
 		currentScope.addSymbol(symbol.acSymbol, false);
-		symbol.acSymbol.doc = makeDocumentation(dec.comment);
+		makeDocumentation(symbol.acSymbol.doc, dec.comment);
 
 		istring lastComment = this.lastComment;
 		this.lastComment = istring.init;
@@ -409,6 +414,7 @@ final class FirstPass : ASTVisitor
 		}
 
 		currentSymbol = currentSymbol.parent;
+		previousSymbol = symbol;
 	}
 
 	mixin visitEnumMember!EnumMember;
@@ -841,11 +847,15 @@ private:
 		currentSymbol.addChild(symbol, true);
 		currentScope.addSymbol(symbol.acSymbol, false);
 		currentSymbol = symbol;
+		pushedSymbolsStack.assumeSafeAppend ~= symbol;
 	}
 
 	void popSymbol()
 	{
+		assert(pushedSymbolsStack.length, "called popSymbol without pushSymbol");
 		currentSymbol = currentSymbol.parent;
+		previousSymbol = pushedSymbolsStack[$ - 1];
+		pushedSymbolsStack.length--;
 	}
 
 	template visitEnumMember(T)
@@ -855,7 +865,7 @@ private:
 			pushSymbol(member.name.text, CompletionKind.enumMember, symbolFile,
 				member.name.index, member.type);
 			scope(exit) popSymbol();
-			currentSymbol.acSymbol.doc = makeDocumentation(member.comment);
+			makeDocumentation(currentSymbol.acSymbol.doc, member.comment);
 		}
 	}
 
@@ -875,7 +885,7 @@ private:
 		else
 			currentSymbol.acSymbol.addChildren(aggregateSymbols[], false);
 		currentSymbol.acSymbol.protection = protection.current;
-		currentSymbol.acSymbol.doc = makeDocumentation(dec.comment);
+		makeDocumentation(currentSymbol.acSymbol.doc, dec.comment);
 
 		istring lastComment = this.lastComment;
 		this.lastComment = istring.init;
@@ -904,7 +914,7 @@ private:
 		currentSymbol.addChild(symbol, true);
 		processParameters(symbol, null, THIS_SYMBOL_NAME, parameters, templateParameters);
 		symbol.acSymbol.protection = protection.current;
-		symbol.acSymbol.doc = makeDocumentation(doc);
+		makeDocumentation(symbol.acSymbol.doc, doc);
 
 		istring lastComment = this.lastComment;
 		this.lastComment = istring.init;
@@ -917,6 +927,7 @@ private:
 			currentSymbol = symbol;
 			functionBody.accept(this);
 			currentSymbol = currentSymbol.parent;
+			previousSymbol = symbol;
 		}
 	}
 
@@ -928,7 +939,7 @@ private:
 		currentSymbol.addChild(symbol, true);
 		symbol.acSymbol.callTip = internString("~this()");
 		symbol.acSymbol.protection = protection.current;
-		symbol.acSymbol.doc = makeDocumentation(doc);
+		makeDocumentation(symbol.acSymbol.doc, doc);
 
 		istring lastComment = this.lastComment;
 		this.lastComment = istring.init;
@@ -941,6 +952,7 @@ private:
 			currentSymbol = symbol;
 			functionBody.accept(this);
 			currentSymbol = currentSymbol.parent;
+			previousSymbol = symbol;
 		}
 	}
 
@@ -1150,14 +1162,85 @@ private:
 			lookups.insert(lookup);
 	}
 
-	DocString makeDocumentation(string documentation)
+	void makeDocumentation(ref DocString into, string documentation)
 	{
 		if (documentation.isDitto)
-			return DocString(lastComment, true);
+		{
+			into = DocString(lastComment, true);
+			into.dittoOf = lastDocStringInstance;
+			lastDocStringInstance = &into;
+		}
 		else
 		{
 			lastComment = internString(documentation);
-			return DocString(lastComment, false);
+			into = DocString(lastComment, false);
+			lastDocStringInstance = &into;
+		}
+	}
+
+	static makeExampleDocumentation(const Unittest block, ref DocString doc)
+	{
+		import dparse.trivia;
+		import std.algorithm;
+		import std.array;
+		import std.string;
+
+		auto tokens = block.tokens;
+		if (tokens.length)
+		{
+			if (block.comment !is null)
+			{
+				auto data = appender!string;
+				data ~= "Examples:\n\n";
+				if (block.comment.length)
+				{
+					data ~= block.comment;
+					data ~= "\n\n";
+				}
+				data ~= "---\n";
+				assert(tokens.length >= 3);
+				auto unittestTok = tokens.countUntil!(t => t.type == tok!"unittest");
+				assert(unittestTok != -1);
+				auto openingTok = tokens[unittestTok .. $].countUntil!(t => t.type == tok!"{");
+				assert(openingTok != -1);
+				openingTok += unittestTok;
+				assert(tokens[$ - 1].type == tok!"}");
+
+				auto codeData = appender!string;
+				foreach (trailingStart; tokens[openingTok].trailingTrivia)
+					codeData ~= trailingStart.text;
+
+				size_t currentLine = size_t.max;
+				foreach (token; tokens[openingTok + 1 .. $ - 1])
+				{
+					currentLine = token.line;
+
+					foreach (leading; token.leadingTrivia)
+						codeData ~= leading.text;
+
+					if (token.text.length)
+						codeData ~= token.text;
+					else
+						codeData ~= str(token.type);
+
+					foreach (trailing; token.trailingTrivia)
+						codeData ~= trailing.text;
+				}
+
+				foreach (leadingEnd; tokens[$ - 1].leadingTrivia)
+					codeData ~= leadingEnd.text;
+
+				data ~= codeData.data.outdent.chompPrefix("\n").chomp("\n");
+
+				data ~= "\n---\n";
+
+				DocString* s = &doc;
+				while (s)
+				{
+					s.examples ~= istring(data.data);
+					s = s.dittoOf;
+				}
+			}
 		}
 	}
 
@@ -1168,7 +1251,10 @@ private:
 	Scope* currentScope;
 
 	/// Current symbol
-	SemanticSymbol* currentSymbol;
+	SemanticSymbol* currentSymbol, previousSymbol;
+
+	/// Stack of semantic symbols, for referencing the previousSymbol from popSymbol
+	SemanticSymbol*[] pushedSymbolsStack;
 
 	/// Path to the file being converted
 	istring symbolFile;
@@ -1185,6 +1271,8 @@ private:
 	/// Last comment for ditto-ing
 	istring lastComment;
 
+	DocString* lastDocStringInstance;
+
 	const Module mod;
 
 	Rebindable!(const ExpressionNode) feExpression;

dsymbol/src/dsymbol/conversion/package.d

@@ -138,10 +138,15 @@ class SimpleParser : Parser
 {
 	override Unittest parseUnittest()
 	{
+		auto start = index;
 		expect(tok!"unittest");
 		if (currentIs(tok!"{"))
 			skipBraces();
-		return allocator.make!Unittest;
+		auto ret = allocator.make!Unittest;
+		ret.tokens = tokens[start .. index];
+		ret.comment = comment;
+		comment = null;
+		return ret;
 	}
 
 	override MissingFunctionBody parseMissingFunctionBody()

dsymbol/src/dsymbol/symbol.d

@@ -434,22 +434,40 @@ struct DocString
 	/// Creates a non-ditto comment.
 	this(istring content)
 	{
-		this.content = content;
+		this.rawContent = content;
 	}
 
 	/// Creates a comment which may have been ditto, but has been resolved.
 	this(istring content, bool ditto)
 	{
-		this.content = content;
+		this.rawContent = content;
 		this.ditto = ditto;
 	}
 
-	alias content this;
+	alias toString this;
+
+	deprecated("use toString to get a full formatted doc string or rawContent for just what is applied directly (or ditto'd) on the function") alias content = rawContent;
 
 	/// Contains the documentation string associated with this symbol, resolves ditto to the previous comment with correct scope.
-	istring content;
+	istring rawContent;
 	/// `true` if the documentation was just a "ditto" comment copying from the previous comment.
 	bool ditto;
+	/// Contains the source code + docstring for each documented unittest example associated with this symbol.
+	istring[] examples;
+
+	// package-private because we don't want to overcomplicate the lifetime in
+	// the public API. (This might point to a broken address later)
+	package(dsymbol) DocString* dittoOf;
+
+	string toString() const @safe pure
+	{
+		import std.algorithm;
+		import std.array;
+
+		return examples.length
+			? (rawContent ~ "\n\n" ~ examples.map!"a.data".join("\n"))
+			: rawContent;
+	}
 }
 
 struct UpdatePair

tests/tc_unittest_docs/expected1.txt

@@ -0,0 +1 @@
+Does foo stuff.\n\nExamples:\n\n---\n// usable with ints\nfoo(1);\n// and with strings!\nif (auto line = readln())\n    foo(line);\n\n// or here\nfoo( 1+2 );\n---\n\nExamples:\n\nsecond usage works too\n\n---\nfoo();\n---\n

tests/tc_unittest_docs/file.d

@@ -0,0 +1,30 @@
+void main()
+{
+    foo(1);
+}
+
+/// Does foo stuff.
+template foo()
+{
+    void foo(int a) {}
+    void foo(string b) {}
+}
+
+///
+unittest
+{
+    // usable with ints
+    foo(1);
+    // and with strings!
+    if (auto line = readln())
+        foo(line);
+
+    // or here
+    foo( 1+2 );
+}
+
+/// second usage works too
+unittest
+{
+    foo();
+}

tests/tc_unittest_docs/run.sh

@@ -0,0 +1,5 @@
+set -e
+set -u
+
+../../bin/dcd-client $1 file.d -d -c20 > actual1.txt
+diff actual1.txt expected1.txt