add ///output directive

dvirtz · Oct 21, 2020 · eebb794 · eebb794
1 parent 0f80aba
commit eebb794
Show file tree

Hide file tree

Showing 9 changed files with 496 additions and 37 deletions.
diff --git a/package.json b/package.json
@@ -30,6 +30,7 @@
     "lerna": "^3.22.1",
     "mocha": "^8.1.3",
     "reveal-md": "^4.1.4",
+    "rewire": "^5.0.0",
     "rollup": "^2.28.2",
     "rollup-plugin-delete": "^2.0.0",
     "rollup-plugin-node-builtins": "^2.1.2",

diff --git a/packages/reveal-compiler-explorer-demo/demo.md b/packages/reveal-compiler-explorer-demo/demo.md
@@ -25,6 +25,7 @@ Ctrl-click* on code to go to compiler explorer
 
 ```cpp []
 ///external
+///output=Hello CE!
 ///execute
 #include <iostream>
 

diff --git a/packages/reveal-compiler-explorer-demo/test/demo.test.js b/packages/reveal-compiler-explorer-demo/test/demo.test.js
@@ -14,7 +14,7 @@ describe("demo presentation", function () {
   const codeInfo = parseMarkdownSync(path, config);
   codeInfo.forEach((info, index) => {
     it(`should have snippet ${index} compiled`, async function () {
-      await assert.doesNotReject(compile(info));
+      await compile(info);
     });
   });
 });
diff --git a/packages/reveal-test/README.md b/packages/reveal-test/README.md
@@ -19,7 +19,7 @@ describe("demo presentation", function () {
   const codeInfo = parseMarkdownSync('presentation.md');
   codeInfo.forEach((info, index) => {
     it(`should have snippet ${index} compiled`, async function () {
-      await assert.doesNotReject(compile(info));
+      await compile(info);
     });
   });
 });
@@ -76,6 +76,14 @@ Returns output from compiling and running the code (if enabled) on success, othe
 
 In addition to the directives mentioned [here](/packages/compiler-explorer-directives/#Directives), the following directives are supported:
 
-### `///failReason=<reason>`
+### `///fails=<reason>`
 
 When given, [`compile`](#compile) is expected to fail with an error containing the given reason.
+
+Cannot be defined together with [`output`](#`///output=<expected>`).
+
+### `///output=<expected>`
+
+When given, [`compile`](#compile) is expected to return the given output.
+
+Cannot be defined together with [`fails`](#`///fails=<reason>`).
diff --git a/packages/reveal-test/dist/reveal-test.cjs b/packages/reveal-test/dist/reveal-test.cjs
@@ -10,30 +10,50 @@ function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'defau
 
 var MarkdownIt__default = /*#__PURE__*/_interopDefaultLegacy(MarkdownIt);
 
-const parseMarkdownImpl = (path, markdown, config) => {
+const parseMarkdownImpl = (markdown, path, config = {}) => {
   const md = new MarkdownIt__default['default']();
   const result = md.parse(markdown, {});
   return result
     .filter(({ type, tag }) => type === 'fence' && tag === 'code')
     .map(({ content, info, map }) => {
+      const [location, error] = (() => {
+        if (path) {
+          return [`${path}:${map[0] + 1}`, message => new Error(`${location}:\n${message}`)];
+        }
+        return [undefined, message => new Error(message)];
+      })();
       config = Object.assign(config, {
-        directives:
-          [['fails=(.*)', (matches, info) => matches.slice(1).forEach(match => info.failReason = match)]]
+        directives: [
+          ['fails=(.*)', (matches, info) => matches.slice(1).forEach(match => {
+            if (info.hasOwnProperty('expectedOutput')) {
+              throw error('cannot have "fails" and "output" together');
+            }
+            info.failReason = match;
+          })],
+          ['output=(.*)', (matches, info) => matches.slice(1).forEach(match => {
+            if (info.hasOwnProperty('failReason')) {
+              throw error('cannot have "fails" and "output" together');
+            }
+            info.expectedOutput = matches[1];
+          })]
+        ]
       });
       const parsed = compilerExplorerDirectives.parseCode(content.replace(/<br\/>/g, ''), info.replace(/(\w+).*/, '$1'), config);
-      parsed.path = `${path}:${map[0] + 1}`;
+      if (location) {
+        parsed.path = location;
+      }
       return parsed;
     });
 };
 
 const parseMarkdown = async (path, config = {}) => {
   const markdown = await fs.promises.readFile(path, 'utf-8');
-  return parseMarkdownImpl(path, markdown, config);
+  return parseMarkdownImpl(markdown, path, config);
 };
 
 const parseMarkdownSync = (path, config = {}) => {
   const markdown = fs.readFileSync(path, 'utf-8');
-  return parseMarkdownImpl(path, markdown, config);
+  return parseMarkdownImpl(markdown, path, config);
 };
 
 const compile = async (info, retryOptions = {}) => {
@@ -43,19 +63,26 @@ const compile = async (info, retryOptions = {}) => {
   const failureMismatch = (output) => {
     return new compilerExplorerDirectives.CompileError(-1, error(`should have failed with '${info.failReason}'${output.length > 0 ? `\nactual output is:\n${output}` : ''}`));
   };
+  const resultPromise = compilerExplorerDirectives.compile(info, retryOptions);
   if (info.failReason) {
     try {
-      const result = await compilerExplorerDirectives.compile(info, retryOptions);
+      const result = await resultPromise;
       throw failureMismatch(result);
-    } catch(err) {
+    } catch (err) {
       if (!err.message.includes(info.failReason)) {
         throw failureMismatch(err.message);
       }
       return err.message;
     }
   } else {
     try {
-      return await compilerExplorerDirectives.compile(info);
+      if (info.hasOwnProperty('expectedOutput')) {
+        const result = await resultPromise;
+        if (result !== info.expectedOutput) {
+          throw new compilerExplorerDirectives.CompileError(-2, error(`output mismatch:\nactual: ${result}\nexpected: ${info.expectedOutput}`));
+        }
+        return result;
+      }
     } catch (err) {
       const code = err.hasOwnProperty('code') ? err.code : -2;
       throw new compilerExplorerDirectives.CompileError(err.code, error(err.message));

diff --git a/packages/reveal-test/src/reveal-test.js b/packages/reveal-test/src/reveal-test.js
@@ -4,30 +4,50 @@ import { parseCode, compile as origCompile, CompileError } from 'compiler-explor
 import MarkdownIt from 'markdown-it';
 import { readFileSync, promises } from 'fs';
 
-const parseMarkdownImpl = (path, markdown, config) => {
+const parseMarkdownImpl = (markdown, path, config = {}) => {
   const md = new MarkdownIt();
   const result = md.parse(markdown, {});
   return result
     .filter(({ type, tag }) => type === 'fence' && tag === 'code')
     .map(({ content, info, map }) => {
+      const [location, error] = (() => {
+        if (path) {
+          return [`${path}:${map[0] + 1}`, message => new Error(`${location}:\n${message}`)];
+        }
+        return [undefined, message => new Error(message)];
+      })();
       config = Object.assign(config, {
-        directives:
-          [['fails=(.*)', (matches, info) => matches.slice(1).forEach(match => info.failReason = match)]]
+        directives: [
+          ['fails=(.*)', (matches, info) => matches.slice(1).forEach(match => {
+            if (info.hasOwnProperty('expectedOutput')) {
+              throw error('cannot have "fails" and "output" together');
+            }
+            info.failReason = match;
+          })],
+          ['output=(.*)', (matches, info) => matches.slice(1).forEach(match => {
+            if (info.hasOwnProperty('failReason')) {
+              throw error('cannot have "fails" and "output" together');
+            }
+            info.expectedOutput = matches[1];
+          })]
+        ]
       });
       const parsed = parseCode(content.replace(/<br\/>/g, ''), info.replace(/(\w+).*/, '$1'), config);
-      parsed.path = `${path}:${map[0] + 1}`;
+      if (location) {
+        parsed.path = location;
+      }
       return parsed;
     });
 };
 
 const parseMarkdown = async (path, config = {}) => {
   const markdown = await promises.readFile(path, 'utf-8');
-  return parseMarkdownImpl(path, markdown, config);
+  return parseMarkdownImpl(markdown, path, config);
 }
 
 const parseMarkdownSync = (path, config = {}) => {
   const markdown = readFileSync(path, 'utf-8');
-  return parseMarkdownImpl(path, markdown, config);
+  return parseMarkdownImpl(markdown, path, config);
 }
 
 const compile = async (info, retryOptions = {}) => {
@@ -37,19 +57,26 @@ const compile = async (info, retryOptions = {}) => {
   const failureMismatch = (output) => {
     return new CompileError(-1, error(`should have failed with '${info.failReason}'${output.length > 0 ? `\nactual output is:\n${output}` : ''}`));
   };
+  const resultPromise = origCompile(info, retryOptions)
   if (info.failReason) {
     try {
-      const result = await origCompile(info, retryOptions);
+      const result = await resultPromise;
       throw failureMismatch(result);
-    } catch(err) {
+    } catch (err) {
       if (!err.message.includes(info.failReason)) {
         throw failureMismatch(err.message);
       }
       return err.message;
     }
   } else {
     try {
-      return await origCompile(info);
+      if (info.hasOwnProperty('expectedOutput')) {
+        const result = await resultPromise;
+        if (result !== info.expectedOutput) {
+          throw new CompileError(-2, error(`output mismatch:\nactual: ${result}\nexpected: ${info.expectedOutput}`));
+        }
+        return result;
+      }
     } catch (err) {
       const code = err.hasOwnProperty('code') ? err.code : -2;
       throw new CompileError(err.code, error(err.message));

diff --git a/packages/reveal-test/test/parse-markdown.test.js b/packages/reveal-test/test/parse-markdown.test.js
@@ -4,9 +4,14 @@ import { parseMarkdownSync, compile } from 'reveal-test';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 import assert from 'assert';
+import rewire from 'rewire';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
+
+const rewired = rewire(join(__dirname, '../dist/reveal-test.cjs'));
+const parseMarkdownImpl = rewired.__get__('parseMarkdownImpl');
+
 const path = join(__dirname, 'test.md');
 const expected = [{
   source: `#include <iostream>
@@ -27,7 +32,8 @@ int main() {
   libs: [],
   execute: true,
   baseUrl: 'https://godbolt.org/',
-  path: `${path}:19`
+  path: `${path}:19`,
+  expectedOutput: 'Hello CE!'
 }, {
   source: `import cblas;
 
@@ -70,7 +76,7 @@ assert(C == [1, 0,
   }],
   execute: false,
   baseUrl: 'https://godbolt.org/',
-  path: `${path}:48`
+  path: `${path}:49`
 }, {
   source: `#include <iostream>
 
@@ -91,7 +97,7 @@ int main() {
   execute: true,
   failReason: `expected ';' before '}' token`,
   baseUrl: 'https://godbolt.org/',
-  path: `${path}:112`
+  path: `${path}:113`
 }, {
   source: `void foo(short i);
 void foo(int i) = delete;
@@ -111,7 +117,7 @@ foo(42);                        // error, deleted function`,
   options: '-O2 -march=haswell -Wall -Wextra -pedantic -Wno-unused-variable -Wno-unused-parameter',
   libs: [],
   baseUrl: 'https://godbolt.org/',
-  path: `${path}:125`,
+  path: `${path}:126`,
   failReason: 'use of deleted function \'void foo(int)\''
 }, {
   source: `template< class ForwardIt, class T >
@@ -125,7 +131,7 @@ void iota( ForwardIt first, ForwardIt last, T value );
   options: '',
   libs: [],
   baseUrl: 'https://godbolt.org/',
-  path: `${path}:143`,
+  path: `${path}:144`,
   failReason: 'Not Found'
 }
 ];
@@ -141,9 +147,24 @@ describe('parseMarkdown', function () {
       assert.deepStrictEqual(info, expected[index]);
     });
 
-    it(`should compile code block #${index}`, async function() {
+    it(`should compile code block #${index}`, async function () {
       this.timeout(10000);
       await assert.doesNotReject(compile(info));
     });
   });
+});
+
+describe('parseMarkdownImpl', function () {
+  it('errors on output and failure defined together', function () {
+    const markdown = `
+\`\`\`ada
+///output=expected
+///fails=failed
+function Square(num : Integer) return Integer is
+begin
+    return num**2;
+end Square;
+`
+    assert.throws(() => parseMarkdownImpl(markdown), /cannot have "fails" and "output" together/);
+  });
 });
diff --git a/packages/reveal-test/test/test.md b/packages/reveal-test/test/test.md
@@ -18,6 +18,7 @@ Ctrl-click (or ⌘-click) on code to go to compiler explorer
 
 ```cpp
 ///external
+///output=Hello CE!
 #include <iostream>
 
 int main() {