Sourcemaps support for RAM

Summary:This rev adds support for production sourcemaps on RAM.

When we inject a module into JSC we use the original `sourceURL` and specify the `startingLineNumber` of the module relative to a "regular" bundle. By doing so, when an error is thrown, JSC will include the provided `sourceURL` as the filename and will use the indicated `startingLineNumber` to figure out on which line the error actually occurred.

To make things a bit simpler and avoid having to deal with columns, we tweak the generated bundle so that each module starts on a new line. Since we cannot assure that each module's code will be on a single line as the minifier might break it on multiple (UglifyJS does so due to a bug on old versions of Chrome), we include on the index the line number that should be used when invoking `JSEvaluateScript`. Since the module length was not being used we replaced the placeholder we have there for the line number.

Reviewed By: javache

Differential Revision: D2997520

fb-gh-sync-id: 3243a489cbb5b48a963f4ccdd98ba63b30f53f3f
shipit-source-id: 3243a489cbb5b48a963f4ccdd98ba63b30f53f3f

React/Executors/RCTJSCExecutor.m

@@ -32,10 +32,10 @@
 
 static NSString *const RCTJSCProfilerEnabledDefaultsKey = @"RCTJSCProfilerEnabled";
 
-// TODO: add lineNo
 typedef struct ModuleData
 {
   uint32_t offset;
+  uint32_t lineNo;
 } ModuleData;
 
 @interface RCTJavaScriptContext : NSObject <RCTInvalidating>
@@ -668,6 +668,14 @@ static void freeModuleData(__unused CFAllocatorRef allocator, void *ptr)
   free(ptr);
 }
 
+static uint32_t readUint32(const void **ptr) {
+  uint32_t data;
+  memcpy(&data, *ptr, sizeof(uint32_t));
+  data = NSSwapLittleIntToHost(data);
+  *ptr += sizeof(uint32_t);
+  return data;
+}
+
 - (NSData *)loadRAMBundle:(NSData *)script
 {
   __weak RCTJSCExecutor *weakSelf = self;
@@ -679,8 +687,9 @@ - (NSData *)loadRAMBundle:(NSData *)script
 
     ModuleData *moduleData = (ModuleData *)CFDictionaryGetValue(strongSelf->_jsModules, moduleName.UTF8String);
     JSStringRef module = JSStringCreateWithUTF8CString((const char *)_bundle.bytes + moduleData->offset);
+    int lineNo = [moduleName isEqual:@""] ? 0 : moduleData->lineNo;
     JSValueRef jsError = NULL;
-    JSValueRef result = JSEvaluateScript(strongSelf->_context.ctx, module, NULL, strongSelf->_bundleURL, NULL, &jsError);
+    JSValueRef result = JSEvaluateScript(strongSelf->_context.ctx, module, NULL, strongSelf->_bundleURL, lineNo, NULL);
 
     CFDictionaryRemoveValue(strongSelf->_jsModules, moduleName.UTF8String);
     JSStringRelease(module);
@@ -706,26 +715,28 @@ - (NSData *)loadRAMBundle:(NSData *)script
   memcpy(&tableLength, bytes + currentOffset, sizeof(tableLength));
   tableLength = NSSwapLittleIntToHost(tableLength);
 
-  uint32_t baseOffset = currentOffset + tableLength;
+  // offset where the code starts on the bundle
+  const uint32_t baseOffset = currentOffset + tableLength;
+
+  // skip table length
   currentOffset += sizeof(baseOffset);
 
-  while (currentOffset < baseOffset) {
-    const char *moduleName = (const char *)bytes + currentOffset;
-    uint32_t offset;
+  // pointer to first byte out of the index
+  const uint8_t *endOfTable = bytes + baseOffset;
 
-    // the space allocated for each module's metada gets freed when the module is injected into JSC on `nativeRequire`
-    ModuleData *moduleData = malloc(sizeof(moduleData));
+  // pointer to current position on table
+  const uint8_t *tablePos = bytes + 2 * sizeof(uint32_t); // skip magic number and table length
+
+  while (tablePos < endOfTable) {
+    const char *moduleName = (const char *)tablePos;
 
-    // skip module name and null byte terminator
-    currentOffset += strlen(moduleName) + 1;
+    // the space allocated for each module's metada gets freed when the module is injected into JSC on `nativeRequire`
+    ModuleData *moduleData = malloc(sizeof(ModuleData));
 
-    // read and save offset
-    memcpy(&offset, bytes + currentOffset, sizeof(offset));
-    offset = NSSwapLittleIntToHost(offset);
-    moduleData->offset = baseOffset + offset;
+    tablePos += strlen(moduleName) + 1; // null byte terminator
 
-    // TODO: replace length with lineNo
-    currentOffset += sizeof(offset) * 2; // skip both offset and lenght
+    moduleData->offset = baseOffset + readUint32((const void **)&tablePos);
+    moduleData->lineNo = readUint32((const void **)&tablePos);
 
     CFDictionarySetValue(_jsModules, moduleName, moduleData);
   }

local-cli/bundle/output/unbundle/as-indexed-file.js

@@ -8,6 +8,7 @@
  */
 'use strict';
 
+const buildUnbundleSourcemap = require('./buildUnbundleSourcemap');
 const fs = require('fs');
 const Promise = require('promise');
 const writeSourceMap = require('./write-sourcemap');
@@ -26,23 +27,27 @@ function saveAsIndexedFile(bundle, options, log) {
   const {
     'bundle-output': bundleOutput,
     'bundle-encoding': encoding,
-    dev,
     'sourcemap-output': sourcemapOutput,
   } = options;
 
   log('start');
-  const {startupCode, modules} = bundle.getUnbundle({minify: !dev});
+  const {startupCode, modules} = bundle.getUnbundle();
   log('finish');
 
   log('Writing unbundle output to:', bundleOutput);
   const writeUnbundle = writeBuffers(
     fs.createWriteStream(bundleOutput),
     buildTableAndContents(startupCode, modules, encoding)
-  );
-
-  writeUnbundle.then(() => log('Done writing unbundle output'));
-
-  return Promise.all([writeUnbundle, writeSourceMap(sourcemapOutput, '', log)]);
+  ).then(() => log('Done writing unbundle output'));
+
+  return Promise.all([
+    writeUnbundle,
+    writeSourceMap(
+      sourcemapOutput,
+      buildUnbundleSourcemap(bundle),
+      log,
+    ),
+  ]);
 }
 
 /* global Buffer: true */
@@ -60,13 +65,16 @@ function writeBuffers(stream, buffers) {
   });
 }
 
-const moduleToBuffer = ({name, code}, encoding) => ({
-  name,
-  buffer: Buffer.concat([
-    Buffer(code, encoding),
-    nullByteBuffer // create \0-terminated strings
-  ])
-});
+function moduleToBuffer(name, code, encoding) {
+  return {
+    name,
+    linesCount: code.split('\n').length,
+    buffer: Buffer.concat([
+      Buffer(code, encoding),
+      nullByteBuffer // create \0-terminated strings
+    ])
+  };
+}
 
 function uInt32Buffer(n) {
   const buffer = Buffer(4);
@@ -82,23 +90,28 @@ function buildModuleTable(buffers) {
   // entry:
   //  - module_id: NUL terminated utf8 string
   //  - module_offset: uint_32 offset into the module string
-  //  - module_length: uint_32 length of the module string, including terminating NUL byte
+  //  - module_line: uint_32 line on which module starts on the bundle
 
   const numBuffers = buffers.length;
 
   const tableLengthBuffer = uInt32Buffer(0);
   let tableLength = 4; // the table length itself, 4 == tableLengthBuffer.length
   let currentOffset = 0;
+  let currentLine = 1;
 
   const offsetTable = [tableLengthBuffer];
   for (let i = 0; i < numBuffers; i++) {
-    const {name, buffer: {length}} = buffers[i];
+    const {name, linesCount, buffer: {length}} = buffers[i];
+
     const entry = Buffer.concat([
       Buffer(i === 0 ? MAGIC_STARTUP_MODULE_ID : name, 'utf8'),
       nullByteBuffer,
       uInt32Buffer(currentOffset),
-      uInt32Buffer(length)
+      uInt32Buffer(currentLine),
     ]);
+
+    currentLine += linesCount - 1;
+
     currentOffset += length;
     tableLength += entry.length;
     offsetTable.push(entry);
@@ -110,14 +123,22 @@ function buildModuleTable(buffers) {
 
 function buildModuleBuffers(startupCode, modules, encoding) {
   return (
-    [moduleToBuffer({name: '', code: startupCode}, encoding)]
-      .concat(modules.map(module => moduleToBuffer(module, encoding)))
+    [moduleToBuffer('', startupCode, encoding, true)].concat(
+      modules.map(module =>
+        moduleToBuffer(
+          module.name,
+          module.code + '\n', // each module starts on a newline
+          encoding,
+        )
+      )
+    )
   );
 }
 
 function buildTableAndContents(startupCode, modules, encoding) {
   const buffers = buildModuleBuffers(startupCode, modules, encoding);
   const table = buildModuleTable(buffers, encoding);
+
   return [fileHeader, table].concat(buffers.map(({buffer}) => buffer));
 }
 

local-cli/bundle/output/unbundle/buildUnbundleSourcemap.js

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+const sourceMap  = require('source-map');
+const SourceMapConsumer = sourceMap.SourceMapConsumer;
+
+/**
+ * Builds the sourcemaps for any type of unbundle provided the Bundle that
+ * contains the modules reachable from the entry point.
+ *
+ * The generated sourcemaps correspond to a regular bundle on which each module
+ * starts on a new line. Depending on the type of unbundle you're using, you
+ * will have to pipe the line number to native and use it when injecting the
+ * module's code into JSC. This way, we'll trick JSC to believe all the code is
+ * on a single big regular bundle where as it could be on an indexed bundle or
+ * as sparsed assets.
+ */
+function buildUnbundleSourcemap(bundle) {
+  const generator = new sourceMap.SourceMapGenerator({});
+  const nonPolyfillModules = bundle.getModules().filter(module =>
+    !module.polyfill
+  );
+
+  let offset = 1;
+  nonPolyfillModules.forEach(module => {
+    if (module.map) { // assets have no sourcemap
+      const consumer = new SourceMapConsumer(module.map);
+      consumer.eachMapping(mapping => {
+        generator.addMapping({
+          original: {
+            line: mapping.originalLine,
+            column: mapping.originalColumn,
+          },
+          generated: {
+            line: mapping.generatedLine + offset,
+            column: mapping.generatedColumn,
+          },
+          source: module.sourcePath,
+        });
+      });
+
+      generator.setSourceContent(module.sourcePath, module.sourceCode);
+    }
+
+    // some modules span more than 1 line
+    offset += module.code.split('\n').length;
+  });
+
+  return generator.toString();
+}
+
+module.exports = buildUnbundleSourcemap;

local-cli/bundle/output/unbundle/write-sourcemap.js

@@ -16,7 +16,7 @@ function writeSourcemap(fileName, contents, log) {
     return Promise.resolve();
   }
   log('Writing sourcemap output to:', fileName);
-  const writeMap = writeFile(fileName, '', null);
+  const writeMap = writeFile(fileName, contents, null);
   writeMap.then(() => log('Done writing sourcemap output'));
   return writeMap;
 }

packager/react-packager/src/Bundler/Bundle.js

@@ -16,9 +16,6 @@ const crypto = require('crypto');
 
 const SOURCEMAPPING_URL = '\n\/\/# sourceMappingURL=';
 
-const getCode = x => x.code;
-const getNameAndCode = ({name, code}) => ({name, code});
-
 class Bundle extends BundleBase {
   constructor({sourceMapUrl, minify} = {}) {
     super();
@@ -40,6 +37,7 @@ class Bundle extends BundleBase {
       map: moduleTransport.map,
       meta: moduleTransport.meta,
       minify: this._minify,
+      polyfill: module.isPolyfill(),
     }).then(({code, map}) => {
       // If we get a map from the transformer we'll switch to a mode
       // were we're combining the source maps as opposed to
@@ -113,20 +111,21 @@ class Bundle extends BundleBase {
     const modules =
       allModules
         .splice(prependedModules, allModules.length - requireCalls - prependedModules);
-    const startupCode = allModules.map(getCode).join('\n');
+    const startupCode = allModules.map(({code}) => code).join('\n');
 
     return {
       startupCode,
-      modules: modules.map(getNameAndCode)
+      modules: modules.map(({name, code, polyfill}) =>
+        ({name, code, polyfill})
+      ),
     };
   }
 
   /**
-   * I found a neat trick in the sourcemap spec that makes it easy
-   * to concat sourcemaps. The `sections` field allows us to combine
-   * the sourcemap easily by adding an offset. Tested on chrome.
-   * Seems like it's not yet in Firefox but that should be fine for
-   * now.
+   * Combine each of the sourcemaps multiple modules have into a single big
+   * one. This works well thanks to a neat trick defined on the sourcemap spec
+   * that makes use of of the `sections` field to combine sourcemaps by adding
+   * an offset. This is supported only by Chrome for now.
    */
   _getCombinedSourceMaps(options) {
     const result = {

packager/react-packager/src/Bundler/__tests__/Bundle-test.js

@@ -123,7 +123,7 @@ describe('Bundle', () => {
       };
 
       const promise = Promise.all(
-        moduleTransports.map(m => bundle.addModule(resolver, null, null, m)))
+        moduleTransports.map(m => bundle.addModule(resolver, null, {isPolyfill: () => false}, m)))
       .then(() => {
         expect(bundle.getModules())
           .toEqual(moduleTransports);
@@ -375,12 +375,19 @@ function resolverFor(code, map) {
   };
 }
 
-function addModule({bundle, code, sourceCode, sourcePath, map, virtual}) {
+function addModule({bundle, code, sourceCode, sourcePath, map, virtual, polyfill}) {
   return bundle.addModule(
     resolverFor(code, map),
     null,
-    null,
-    createModuleTransport({code, sourceCode, sourcePath, map, virtual})
+    {isPolyfill: () => polyfill},
+    createModuleTransport({
+      code,
+      sourceCode,
+      sourcePath,
+      map,
+      virtual,
+      polyfill,
+    }),
   );
 }
 

packager/react-packager/src/Bundler/__tests__/Bundler-test.js

@@ -204,6 +204,7 @@ describe('Bundler', function() {
           transform: {
             dev: true,
             hot: false,
+            generateSourceMaps: false,
             projectRoots,
           }
         },