Skip to content

Commit

Permalink
feat(offline compiler): a replacement for tsc that compiles templates
Browse files Browse the repository at this point in the history
  • Loading branch information
alexeagle committed Apr 29, 2016
1 parent 33e53c9 commit 78946fe
Show file tree
Hide file tree
Showing 17 changed files with 743 additions and 23 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ tmp
*.js.deps
*.js.map

# Files created by the template compiler
**/*.ngfactory.ts

# Or type definitions we mirror from github
# (NB: these lines are removed in publish-build-artifacts.sh)
**/typings/**/*.d.ts
Expand Down
60 changes: 49 additions & 11 deletions gulpfile.js
Original file line number Diff line number Diff line change
Expand Up @@ -452,22 +452,40 @@ gulp.task('serve.e2e.dart', ['build.js.cjs'], function(neverDone) {
// ------------------
// CI tests suites

function runKarma(configFile, done) {
function execProcess(name, args, done) {
var exec = require('child_process').exec;

var cmd = process.platform === 'win32' ? 'node_modules\\.bin\\karma run ' :
'node node_modules/.bin/karma run ';
cmd += configFile;
exec(cmd, function(e, stdout) {
var cmd = process.platform === 'win32' ? 'node_modules\\.bin\\' + name + ' ' :
'node node_modules/.bin/' + name + ' ';
cmd += args;
exec(cmd, done);
}
function runKarma(configFile, done) {
execProcess('karma', 'run ' + configFile, function(e, stdout) {
// ignore errors, we don't want to fail the build in the interactive (non-ci) mode
// karma server will print all test failures
done();
});
}

// Gulp-typescript doesn't work with typescript@next:
// https://github.com/ivogabe/gulp-typescript/issues/331
function runTsc(project, done) {
execProcess('tsc', '-p ' + project, function(e, stdout, stderr) {
if (e) {
console.log(stdout);
console.error(stderr);
done(e);
} else {
done();
}
});
}

gulp.task('test.js', function(done) {
runSequence('test.unit.tools/ci', 'test.transpiler.unittest', 'test.unit.js/ci',
'test.unit.cjs/ci', 'test.typings', 'check-public-api', sequenceComplete(done));
'test.unit.cjs/ci', 'test.compiler_cli', 'test.typings', 'check-public-api',
sequenceComplete(done));
});

gulp.task('test.dart', function(done) {
Expand Down Expand Up @@ -768,7 +786,7 @@ gulp.task('!checkAndReport.payload.js', function() {
{
failConditions: PAYLOAD_TESTS_CONFIG.ts[packaging].sizeLimits,
prefix: caseName + '_' + packaging
})
});
}

return PAYLOAD_TESTS_CONFIG.ts.cases.reduce(function(sizeReportingStreams, caseName) {
Expand Down Expand Up @@ -1026,6 +1044,26 @@ gulp.task('!test.typings',
gulp.task('test.typings', ['build.js.cjs'],
function(done) { runSequence('!test.typings', sequenceComplete(done)); });

gulp.task('!build.compiler_cli', ['build.js.cjs'],
function(done) { runTsc('tools/compiler_cli/src', done); });

gulp.task('!test.compiler_cli.codegen', function(done) {
try {
require('./dist/js/cjs/compiler_cli')
.main("tools/compiler_cli/test")
.then(function() { done(); })
.catch(function(rej) { done(new Error(rej)); });
} catch (err) {
done(err);
}
});

// End-to-end test for compiler CLI.
// Calls the compiler using its command-line interface, then compiles the app with the codegen.
// TODO(alexeagle): wire up the playground tests with offline compilation, similar to dart.
gulp.task('test.compiler_cli', ['!build.compiler_cli'],
function(done) { runSequence('!test.compiler_cli.codegen', sequenceComplete(done)); });

// -----------------
// orchestrated targets

Expand Down Expand Up @@ -1091,7 +1129,7 @@ gulp.task('!build.tools', function() {
var sourcemaps = require('gulp-sourcemaps');
var tsc = require('gulp-typescript');

var stream = gulp.src(['tools/**/*.ts'])
var stream = gulp.src(['tools/**/*.ts', '!tools/compiler_cli/**'])
.pipe(sourcemaps.init())
.pipe(tsc({
target: 'ES5',
Expand Down Expand Up @@ -1512,7 +1550,7 @@ gulp.on('task_start', (e) => {
analytics.buildSuccess('gulp <startup>', process.uptime() * 1000);
}

analytics.buildStart('gulp ' + e.task)
analytics.buildStart('gulp ' + e.task);
});
gulp.on('task_stop', (e) => {analytics.buildSuccess('gulp ' + e.task, e.duration * 1000)});
gulp.on('task_err', (e) => {analytics.buildError('gulp ' + e.task, e.duration * 1000)});
gulp.on('task_stop', (e) => { analytics.buildSuccess('gulp ' + e.task, e.duration * 1000); });
gulp.on('task_err', (e) => { analytics.buildError('gulp ' + e.task, e.duration * 1000); });
31 changes: 19 additions & 12 deletions tools/build/linknodemodules.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,21 @@ var fs = require('fs');
var path = require('path');

module.exports = function(gulp, plugins, config) {
function symlink(relativeFolder, linkDir) {
var sourceDir = path.join('..', relativeFolder);
if (!fs.existsSync(linkDir)) {
console.log('creating link', linkDir, sourceDir);
try {
fs.symlinkSync(sourceDir, linkDir, 'dir');
}
catch(e) {
var sourceDir = path.join(config.dir, relativeFolder);
console.log('linking failed: trying to hard copy', linkDir, sourceDir);
copyRecursiveSync(sourceDir, linkDir);
}
}
}

return function() {
var nodeModulesDir = path.join(config.dir, 'node_modules');
if (!fs.existsSync(nodeModulesDir)) {
Expand All @@ -11,20 +26,12 @@ module.exports = function(gulp, plugins, config) {
if (relativeFolder === 'node_modules') {
return;
}
var sourceDir = path.join('..', relativeFolder);

var linkDir = path.join(nodeModulesDir, relativeFolder);
if (!fs.existsSync(linkDir)) {
console.log('creating link', linkDir, sourceDir);
try {
fs.symlinkSync(sourceDir, linkDir, 'dir');
}
catch(e) {
var sourceDir = path.join(config.dir, relativeFolder);
console.log('linking failed: trying to hard copy', linkDir, sourceDir);
copyRecursiveSync(sourceDir, linkDir);
}
}
symlink(relativeFolder, linkDir);
});
// Also symlink tools we release independently to NPM, so tests can require metadata, etc.
symlink('../../tools/metadata', path.join(nodeModulesDir, 'ts-metadata-collector'));
};
};

Expand Down
72 changes: 72 additions & 0 deletions tools/compiler_cli/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
# Angular Template Compiler

Angular applications are built with templates, which may be `.html` or `.css` files,
or may be inline `template` attributes on Decorators like `@Component`.

These templates are compiled into executable JS at application runtime (except in `interpretation` mode).
This compilation can occur on the client, but it results in slower bootstrap time, and also
requires that the compiler be included in the code downloaded to the client.

You can produce smaller, faster applications by running Angular's compiler as a build step,
and then downloading only the executable JS to the client.

## Configuration

The `tsconfig.json` file is expected to contain an additional configuration block:
```
"angularCompilerOptions": {
"genDir": "."
}
```
the `genDir` option controls the path (relative to `tsconfig.json`) where the generated file tree
will be written. More options may be added as we implement more features.

We recommend you avoid checking generated files into version control. This permits a state where
the generated files in the repository were created from sources that were never checked in,
making it impossible to reproduce the current state. Also, your changes will effectively appear
twice in code reviews, with the generated version inscrutible by the reviewer.

In TypeScript 1.8, the generated sources will have to be written alongside your originals,
so set `genDir` to the same location as your files (typicially the same as `rootDir`).
Add `**/*.ngfactory.ts` to your `.gitignore` or other mechanism for your version control system.

In TypeScript 1.9 and above, you can add a generated folder into your application,
such as `codegen`. Using the `rootDirs` option, you can allow relative imports like
`import {} from './foo.ngfactory'` even though the `src` and `codegen` trees are distinct.
Add `**/codegen` to your `.gitignore` or similar.

Note that in the second option, TypeScript will emit the code into two parallel directories
as well. This is by design, see https://github.com/Microsoft/TypeScript/issues/8245.
This makes the configuration of your runtime module loader more complex, so we don't recommend
this option yet.

See the example in the `test/` directory for a working example.

## Compiler CLI

This program mimics the TypeScript tsc command line. It accepts a `-p` flag which points to a
`tsconfig.json` file, or a directory containing one.

This CLI is intended for demos, prototyping, or for users with simple build systems
that run bare `tsc`.

Users with a build system should expect an Angular 2 template plugin. Such a plugin would be
based on the `index.ts` in this directory, but should share the TypeScript compiler instance
with the one already used in the plugin for TypeScript typechecking and emit.

## Design
At a high level, this program
- collects static metadata about the sources using the `ts-metadata-collector` package in angular2
- uses the `OfflineCompiler` from `angular2/src/compiler/compiler` to codegen additional `.ts` files
- these `.ts` files are written to the `genDir` path, then compiled together with the application.

## For developers
Run the compiler from source:
```
# Build angular2
gulp build.js.cjs
# Build the compiler
./node_modules/.bin/tsc -p tools/compiler_cli/src
# Run it on the test project
node ./dist/js/cjs/compiler_cli -p tools/compiler_cli/test
```
142 changes: 142 additions & 0 deletions tools/compiler_cli/src/codegen.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
/**
* Transform template html and css into executable code.
* Intended to be used in a build step.
*/
import * as ts from 'typescript';
import * as path from 'path';

import * as compiler from 'angular2/compiler';
import {StaticReflector} from 'angular2/src/compiler/static_reflector';
import {CompileMetadataResolver} from 'angular2/src/compiler/metadata_resolver';
import {HtmlParser} from 'angular2/src/compiler/html_parser';
import {DirectiveNormalizer} from 'angular2/src/compiler/directive_normalizer';
import {Lexer} from 'angular2/src/compiler/expression_parser/lexer';
import {Parser} from 'angular2/src/compiler/expression_parser/parser';
import {TemplateParser} from 'angular2/src/compiler/template_parser';
import {DomElementSchemaRegistry} from 'angular2/src/compiler/schema/dom_element_schema_registry';
import {StyleCompiler} from 'angular2/src/compiler/style_compiler';
import {ViewCompiler} from 'angular2/src/compiler/view_compiler/view_compiler';
import {TypeScriptEmitter} from 'angular2/src/compiler/output/ts_emitter';
import {RouterLinkTransform} from 'angular2/src/router/directives/router_link_transform';
import {Parse5DomAdapter} from 'angular2/platform/server';

import {MetadataCollector} from 'ts-metadata-collector';
import {NodeReflectorHost} from './reflector_host';
import {wrapCompilerHost, CodeGeneratorHost} from './compiler_host';

const SOURCE_EXTENSION = /\.[jt]s$/;
const PREAMBLE = `/**
* This file is generated by the Angular 2 template compiler.
* Do not edit.
*/
`;

export interface AngularCompilerOptions {
// Absolute path to a directory where generated file structure is written
genDir: string;
}

export class CodeGenerator {
constructor(private ngOptions: AngularCompilerOptions, private basePath: string,
public program: ts.Program, public host: CodeGeneratorHost,
private staticReflector: StaticReflector, private resolver: CompileMetadataResolver,
private compiler: compiler.OfflineCompiler,
private reflectorHost: NodeReflectorHost) {}

private generateSource(metadatas: compiler.CompileDirectiveMetadata[]) {
const normalize = (metadata: compiler.CompileDirectiveMetadata) => {
const directiveType = metadata.type.runtime;
const directives = this.resolver.getViewDirectivesMetadata(directiveType);
const pipes = this.resolver.getViewPipesMetadata(directiveType);
return new compiler.NormalizedComponentWithViewDirectives(metadata, directives, pipes);
};

return this.compiler.compileTemplates(metadatas.map(normalize));
}

private readComponents(absSourcePath: string) {
const result: Promise<compiler.CompileDirectiveMetadata>[] = [];
const metadata = this.staticReflector.getModuleMetadata(absSourcePath);
if (!metadata) {
console.log(`WARNING: no metadata found for ${absSourcePath}`);
return result;
}

const symbols = Object.keys(metadata['metadata']);
if (!symbols || !symbols.length) {
return result;
}
for (const symbol of symbols) {
const staticType = this.reflectorHost.findDeclaration(absSourcePath, symbol, absSourcePath);
let directive: compiler.CompileDirectiveMetadata;
directive = this.resolver.maybeGetDirectiveMetadata(<any>staticType);

if (!directive || !directive.isComponent) {
continue;
}
result.push(this.compiler.normalizeDirectiveMetadata(directive));
}
return result;
}

codegen() {
Parse5DomAdapter.makeCurrent();
const generateOneFile = (absSourcePath: string) =>
Promise.all(this.readComponents(absSourcePath))
.then((metadatas: compiler.CompileDirectiveMetadata[]) => {
if (!metadatas || !metadatas.length) {
return;
}
const generated = this.generateSource(metadatas);
const sourceFile = this.program.getSourceFile(absSourcePath);

// Write codegen in a directory structure matching the sources.
// TODO(alexeagle): maybe use generated.moduleUrl instead of hardcoded ".ngfactory.ts"
// TODO(alexeagle): relativize paths by the rootDirs option
const emitPath =
path.join(this.ngOptions.genDir, path.relative(this.basePath, absSourcePath))
.replace(SOURCE_EXTENSION, '.ngfactory.ts');
this.host.writeFile(emitPath, PREAMBLE + generated.source, false, () => {},
[sourceFile]);
})
.catch((e) => { console.error(e.stack); });

return Promise.all(this.program.getRootFileNames()
.filter(f => !/\.ngfactory\.ts$/.test(f))
.map(generateOneFile));
}

static create(ngOptions: AngularCompilerOptions, parsed: ts.ParsedCommandLine, basePath: string,
compilerHost: ts.CompilerHost):
{errors?: ts.Diagnostic[], generator?: CodeGenerator} {
const program = ts.createProgram(parsed.fileNames, parsed.options, compilerHost);
const errors = program.getOptionsDiagnostics();
if (errors && errors.length) {
return {errors};
}

const metadataCollector = new MetadataCollector();
const reflectorHost =
new NodeReflectorHost(program, metadataCollector, compilerHost, parsed.options);
const xhr: compiler.XHR = {get: (s: string) => Promise.resolve(compilerHost.readFile(s))};
const urlResolver: compiler.UrlResolver = compiler.createOfflineCompileUrlResolver();
const staticReflector = new StaticReflector(reflectorHost);
const htmlParser = new HtmlParser();
const normalizer = new DirectiveNormalizer(xhr, urlResolver, htmlParser);
const parser = new Parser(new Lexer());
const tmplParser = new TemplateParser(parser, new DomElementSchemaRegistry(), htmlParser,
/*console*/ null, [new RouterLinkTransform(parser)]);
const offlineCompiler = new compiler.OfflineCompiler(
normalizer, tmplParser, new StyleCompiler(urlResolver),
new ViewCompiler(new compiler.CompilerConfig(true, true, true)), new TypeScriptEmitter());
const resolver = new CompileMetadataResolver(
new compiler.DirectiveResolver(staticReflector), new compiler.PipeResolver(staticReflector),
new compiler.ViewResolver(staticReflector), null, null, staticReflector);

return {
generator: new CodeGenerator(ngOptions, basePath, program,
wrapCompilerHost(compilerHost, parsed.options), staticReflector,
resolver, offlineCompiler, reflectorHost)
};
}
}
Loading

0 comments on commit 78946fe

Please sign in to comment.