Use `fastfs.readWhile` to parse module docblocks

Summary:
Uses `fastfs.readWhile` to build the haste map

- cuts the time needed to build the haste map in half
- we don’t need to allocate memory for all JS files in the tree
- we only read in as much as necessary during startup
- we only read files completely that are part of the bundle
- we will be able to move the transform before dependency extraction

public

Reviewed By: martinbigio

Differential Revision: D2890933

fb-gh-sync-id: 5fef6b53458e8bc95d0251d0bcf16821581a3362

react-packager/src/DependencyResolver/Module.js

@@ -43,7 +43,7 @@ class Module {
     return this._cache.get(
       this.path,
       'isHaste',
-      () => this.read().then(data => !!data.id)
+      () => this._readDocBlock().then(data => !!data.id)
     );
   }
 
@@ -55,9 +55,9 @@ class Module {
     return this._cache.get(
       this.path,
       'name',
-      () => this.read().then(data => {
+      () => this._readDocBlock().then(({id}) => {
-        if (data.id) {
+        if (id) {
-          return data.id;
+          return id;
         }
 
         const p = this.getPackage();
@@ -103,32 +103,52 @@ class Module {
     this._cache.invalidate(this.path);
   }
 
-  read() {
+  _parseDocBlock(docBlock) {
-    if (!this._reading) {
+    // Extract an id for the module if it's using @providesModule syntax
-      this._reading = this._fastfs.readFile(this.path).then(content => {
-        const data = {};
-
-        // Set an id on the module if it's using @providesModule syntax
     // and if it's NOT in node_modules (and not a whitelisted node_module).
     // This handles the case where a project may have a dep that has @providesModule
     // docblock comments, but doesn't want it to conflict with whitelisted @providesModule
     // modules, such as react-haste, fbjs-haste, or react-native or with non-dependency,
     // project-specific code that is using @providesModule.
-        const moduleDocBlock = docblock.parseAsObject(content);
+    const moduleDocBlock = docblock.parseAsObject(docBlock);
-        if (!this._depGraphHelpers.isNodeModulesDir(this.path) &&
+    const provides = moduleDocBlock.providesModule || moduleDocBlock.provides;
-            (moduleDocBlock.providesModule || moduleDocBlock.provides)) {
+
-          data.id = /^(\S*)/.exec(
+    const id = provides && !this._depGraphHelpers.isNodeModulesDir(this.path)
-            moduleDocBlock.providesModule || moduleDocBlock.provides
+        ? /^\S+/.exec(provides)[0]
-          )[1];
+        : undefined;
+    return [id, moduleDocBlock];
   }
 
+  _readDocBlock() {
+    const reading = this._reading || this._docBlock;
+    if (reading) {
+      return reading;
+    }
+    this._docBlock = this._fastfs.readWhile(this.path, whileInDocBlock)
+      .then(docBlock => {
+        const [id] = this._parseDocBlock(docBlock);
+        return {id};
+      });
+    return this._docBlock;
+  }
+
+  read() {
+    if (this._reading) {
+      return this._reading;
+    }
+
+    this._reading = this._fastfs.readFile(this.path).then(content => {
+      const [id, moduleDocBlock] = this._parseDocBlock(content);
+
       // Ignore requires in JSON files or generated code. An example of this
       // is prebuilt files like the SourceMap library.
       if (this.isJSON() || 'extern' in moduleDocBlock) {
-          data.dependencies = [];
+        return {
-          data.asyncDependencies = [];
+          id,
-          data.code = content;
+          dependencies: [],
-          return data;
+          asyncDependencies: [],
+          code: content,
+        };
       } else {
         const transformCode = this._transformCode;
         const codePromise = transformCode
@@ -137,14 +157,15 @@ class Module {
 
         return codePromise.then(({code, dependencies, asyncDependencies}) => {
           const {deps} = this._extractor(code);
-            data.dependencies = dependencies || deps.sync;
+          return {
-            data.asyncDependencies = asyncDependencies || deps.async;
+            id,
-            data.code = code;
+            code,
-            return data;
+            dependencies: dependencies || deps.sync,
+            asyncDependencies: asyncDependencies || deps.async,
+          };
         });
       }
     });
-    }
 
     return this._reading;
   }
@@ -181,4 +202,19 @@ class Module {
   }
 }
 
+function whileInDocBlock(chunk, i, result) {
+  // consume leading whitespace
+  if (!/\S/.test(result)) {
+    return true;
+  }
+
+  // check for start of doc block
+  if (!/^\s*\/(\*{2}|\*?$)/.test(result)) {
+    return false;
+  }
+
+  // check for end of doc block
+  return !/\*\//.test(result);
+}
+
 module.exports = Module;

react-packager/src/DependencyResolver/__tests__/Module-test.js

@@ -69,6 +69,104 @@ describe('Module', () => {
     fastfs.build().then(done);
   });
 
+  describe('Module ID', () => {
+    const moduleId = 'arbitraryModule';
+    const source =
+    `/**
+       * @providesModule ${moduleId}
+       */
+    `;
+
+    let module;
+    beforeEach(() => {
+      module = createModule();
+    });
+
+    describe('@providesModule annotations', () => {
+      beforeEach(() => {
+        mockIndexFile(source);
+      });
+
+      pit('extracts the module name from the header', () =>
+        module.getName().then(name => expect(name).toEqual(moduleId))
+      );
+
+      pit('identifies the module as haste module', () =>
+        module.isHaste().then(isHaste => expect(isHaste).toBe(true))
+      );
+
+      pit('does not transform the file in order to access the name', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).getName()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+
+      pit('does not transform the file in order to access the haste status', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).isHaste()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+    });
+
+    describe('@provides annotations', () => {
+      beforeEach(() => {
+        mockIndexFile(source.replace(/@providesModule/, '@provides'));
+      });
+
+      pit('extracts the module name from the header if it has a @provides annotation', () =>
+        module.getName().then(name => expect(name).toEqual(moduleId))
+      );
+
+      pit('identifies the module as haste module', () =>
+        module.isHaste().then(isHaste => expect(isHaste).toBe(true))
+      );
+
+      pit('does not transform the file in order to access the name', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).getName()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+
+      pit('does not transform the file in order to access the haste status', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).isHaste()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+    });
+
+    describe('no annotation', () => {
+      beforeEach(() => {
+        mockIndexFile('arbitrary(code);');
+      });
+
+      pit('uses the file name as module name', () =>
+        module.getName().then(name => expect(name).toEqual(fileName))
+      );
+
+      pit('does not identify the module as haste module', () =>
+        module.isHaste().then(isHaste => expect(isHaste).toBe(false))
+      );
+
+      pit('does not transform the file in order to access the name', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).getName()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+
+      pit('does not transform the file in order to access the haste status', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).isHaste()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+    });
+  });
+
   describe('Async Dependencies', () => {
     function expectAsyncDependenciesToEqual(expected) {
       const module = createModule();

react-packager/src/__mocks__/fs.js

@@ -112,6 +112,56 @@ fs.stat.mockImpl(function(filepath, callback) {
   }
 });
 
+const noop = () => {};
+fs.open.mockImpl(function(path) {
+  const callback = arguments[arguments.length - 1] || noop;
+  let data, error, fd;
+  try {
+    data = getToNode(path);
+  } catch (e) {
+    error = e;
+  }
+
+  if (error || data == null) {
+    error = Error(`ENOENT: no such file or directory, open ${path}`);
+  }
+  if (data != null) {
+    /* global Buffer: true */
+    fd = {
+      buffer: new Buffer(data, 'utf8'),
+      position: 0,
+    };
+  }
+
+  callback(error, fd);
+});
+
+fs.read.mockImpl((fd, buffer, writeOffset, length, position, callback = noop) => {
+  let bytesWritten;
+  try {
+    if (position == null || position < 0) {
+      ({position} = fd);
+    }
+    bytesWritten =
+      fd.buffer.copy(buffer, writeOffset, position, position + length);
+    fd.position = position + bytesWritten;
+  } catch (e) {
+    callback(Error('invalid argument'));
+    return;
+  }
+  callback(null, bytesWritten, buffer);
+});
+
+fs.close.mockImpl((fd, callback = noop) => {
+  try {
+    fd.buffer = fs.position = undefined;
+  } catch (e) {
+    callback(Error('invalid argument'));
+    return;
+  }
+  callback(null);
+});
+
 var filesystem;
 
 fs.__setMockFilesystem = function(object) {