Use `fastfs.readWhile` to parse module docblocks

Summary:
Uses `fastfs.readWhile` to build the haste map

- cuts the time needed to build the haste map in half
- we don’t need to allocate memory for all JS files in the tree
- we only read in as much as necessary during startup
- we only read files completely that are part of the bundle
- we will be able to move the transform before dependency extraction

public

Reviewed By: martinbigio

Differential Revision: D2890933

fb-gh-sync-id: 5fef6b53458e8bc95d0251d0bcf16821581a3362

react-packager/src/DependencyResolver/Module.js

@@ -43,7 +43,7 @@ class Module {
     return this._cache.get(
       this.path,
       'isHaste',
-      () => this.read().then(data => !!data.id)
+      () => this._readDocBlock().then(data => !!data.id)
     );
   }
 
@@ -55,9 +55,9 @@ class Module {
     return this._cache.get(
       this.path,
       'name',
-      () => this.read().then(data => {
+      () => this._readDocBlock().then(({id}) => {
-        if (data.id) {
+        if (id) {
-          return data.id;
+          return id;
         }
 
         const p = this.getPackage();
@@ -103,48 +103,69 @@ class Module {
     this._cache.invalidate(this.path);
   }
 
-  read() {
+  _parseDocBlock(docBlock) {
-    if (!this._reading) {
+    // Extract an id for the module if it's using @providesModule syntax
-      this._reading = this._fastfs.readFile(this.path).then(content => {
+    // and if it's NOT in node_modules (and not a whitelisted node_module).
-        const data = {};
+    // This handles the case where a project may have a dep that has @providesModule
+    // docblock comments, but doesn't want it to conflict with whitelisted @providesModule
+    // modules, such as react-haste, fbjs-haste, or react-native or with non-dependency,
+    // project-specific code that is using @providesModule.
+    const moduleDocBlock = docblock.parseAsObject(docBlock);
+    const provides = moduleDocBlock.providesModule || moduleDocBlock.provides;
 
-        // Set an id on the module if it's using @providesModule syntax
+    const id = provides && !this._depGraphHelpers.isNodeModulesDir(this.path)
-        // and if it's NOT in node_modules (and not a whitelisted node_module).
+        ? /^\S+/.exec(provides)[0]
-        // This handles the case where a project may have a dep that has @providesModule
+        : undefined;
-        // docblock comments, but doesn't want it to conflict with whitelisted @providesModule
+    return [id, moduleDocBlock];
-        // modules, such as react-haste, fbjs-haste, or react-native or with non-dependency,
+  }
-        // project-specific code that is using @providesModule.
-        const moduleDocBlock = docblock.parseAsObject(content);
-        if (!this._depGraphHelpers.isNodeModulesDir(this.path) &&
-            (moduleDocBlock.providesModule || moduleDocBlock.provides)) {
-          data.id = /^(\S*)/.exec(
-            moduleDocBlock.providesModule || moduleDocBlock.provides
-          )[1];
-        }
 
-        // Ignore requires in JSON files or generated code. An example of this
+  _readDocBlock() {
-        // is prebuilt files like the SourceMap library.
+    const reading = this._reading || this._docBlock;
-        if (this.isJSON() || 'extern' in moduleDocBlock) {
+    if (reading) {
-          data.dependencies = [];
+      return reading;
-          data.asyncDependencies = [];
-          data.code = content;
-          return data;
-        } else {
-          const transformCode = this._transformCode;
-          const codePromise = transformCode
-              ? transformCode(this, content)
-              : Promise.resolve({code: content});
-
-          return codePromise.then(({code, dependencies, asyncDependencies}) => {
-            const {deps} = this._extractor(code);
-            data.dependencies = dependencies || deps.sync;
-            data.asyncDependencies = asyncDependencies || deps.async;
-            data.code = code;
-            return data;
-          });
-        }
-      });
     }
+    this._docBlock = this._fastfs.readWhile(this.path, whileInDocBlock)
+      .then(docBlock => {
+        const [id] = this._parseDocBlock(docBlock);
+        return {id};
+      });
+    return this._docBlock;
+  }
+
+  read() {
+    if (this._reading) {
+      return this._reading;
+    }
+
+    this._reading = this._fastfs.readFile(this.path).then(content => {
+      const [id, moduleDocBlock] = this._parseDocBlock(content);
+
+      // Ignore requires in JSON files or generated code. An example of this
+      // is prebuilt files like the SourceMap library.
+      if (this.isJSON() || 'extern' in moduleDocBlock) {
+        return {
+          id,
+          dependencies: [],
+          asyncDependencies: [],
+          code: content,
+        };
+      } else {
+        const transformCode = this._transformCode;
+        const codePromise = transformCode
+            ? transformCode(this, content)
+            : Promise.resolve({code: content});
+
+        return codePromise.then(({code, dependencies, asyncDependencies}) => {
+          const {deps} = this._extractor(code);
+          return {
+            id,
+            code,
+            dependencies: dependencies || deps.sync,
+            asyncDependencies: asyncDependencies || deps.async,
+          };
+        });
+      }
+    });
 
     return this._reading;
   }
@@ -181,4 +202,19 @@ class Module {
   }
 }
 
+function whileInDocBlock(chunk, i, result) {
+  // consume leading whitespace
+  if (!/\S/.test(result)) {
+    return true;
+  }
+
+  // check for start of doc block
+  if (!/^\s*\/(\*{2}|\*?$)/.test(result)) {
+    return false;
+  }
+
+  // check for end of doc block
+  return !/\*\//.test(result);
+}
+
 module.exports = Module;

react-packager/src/DependencyResolver/__tests__/Module-test.js

@@ -69,6 +69,104 @@ describe('Module', () => {
     fastfs.build().then(done);
   });
 
+  describe('Module ID', () => {
+    const moduleId = 'arbitraryModule';
+    const source =
+    `/**
+       * @providesModule ${moduleId}
+       */
+    `;
+
+    let module;
+    beforeEach(() => {
+      module = createModule();
+    });
+
+    describe('@providesModule annotations', () => {
+      beforeEach(() => {
+        mockIndexFile(source);
+      });
+
+      pit('extracts the module name from the header', () =>
+        module.getName().then(name => expect(name).toEqual(moduleId))
+      );
+
+      pit('identifies the module as haste module', () =>
+        module.isHaste().then(isHaste => expect(isHaste).toBe(true))
+      );
+
+      pit('does not transform the file in order to access the name', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).getName()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+
+      pit('does not transform the file in order to access the haste status', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).isHaste()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+    });
+
+    describe('@provides annotations', () => {
+      beforeEach(() => {
+        mockIndexFile(source.replace(/@providesModule/, '@provides'));
+      });
+
+      pit('extracts the module name from the header if it has a @provides annotation', () =>
+        module.getName().then(name => expect(name).toEqual(moduleId))
+      );
+
+      pit('identifies the module as haste module', () =>
+        module.isHaste().then(isHaste => expect(isHaste).toBe(true))
+      );
+
+      pit('does not transform the file in order to access the name', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).getName()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+
+      pit('does not transform the file in order to access the haste status', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).isHaste()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+    });
+
+    describe('no annotation', () => {
+      beforeEach(() => {
+        mockIndexFile('arbitrary(code);');
+      });
+
+      pit('uses the file name as module name', () =>
+        module.getName().then(name => expect(name).toEqual(fileName))
+      );
+
+      pit('does not identify the module as haste module', () =>
+        module.isHaste().then(isHaste => expect(isHaste).toBe(false))
+      );
+
+      pit('does not transform the file in order to access the name', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).getName()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+
+      pit('does not transform the file in order to access the haste status', () => {
+        const transformCode =
+          jest.genMockFn().mockReturnValue(Promise.resolve());
+        return createModule({transformCode}).isHaste()
+          .then(() => expect(transformCode).not.toBeCalled());
+      });
+    });
+  });
+
   describe('Async Dependencies', () => {
     function expectAsyncDependenciesToEqual(expected) {
       const module = createModule();

react-packager/src/__mocks__/fs.js

@@ -112,6 +112,56 @@ fs.stat.mockImpl(function(filepath, callback) {
   }
 });
 
+const noop = () => {};
+fs.open.mockImpl(function(path) {
+  const callback = arguments[arguments.length - 1] || noop;
+  let data, error, fd;
+  try {
+    data = getToNode(path);
+  } catch (e) {
+    error = e;
+  }
+
+  if (error || data == null) {
+    error = Error(`ENOENT: no such file or directory, open ${path}`);
+  }
+  if (data != null) {
+    /* global Buffer: true */
+    fd = {
+      buffer: new Buffer(data, 'utf8'),
+      position: 0,
+    };
+  }
+
+  callback(error, fd);
+});
+
+fs.read.mockImpl((fd, buffer, writeOffset, length, position, callback = noop) => {
+  let bytesWritten;
+  try {
+    if (position == null || position < 0) {
+      ({position} = fd);
+    }
+    bytesWritten =
+      fd.buffer.copy(buffer, writeOffset, position, position + length);
+    fd.position = position + bytesWritten;
+  } catch (e) {
+    callback(Error('invalid argument'));
+    return;
+  }
+  callback(null, bytesWritten, buffer);
+});
+
+fs.close.mockImpl((fd, callback = noop) => {
+  try {
+    fd.buffer = fs.position = undefined;
+  } catch (e) {
+    callback(Error('invalid argument'));
+    return;
+  }
+  callback(null);
+});
+
 var filesystem;
 
 fs.__setMockFilesystem = function(object) {