Add ability to load “unbundles” to android

Summary:
public

This adds the ability to load “unbundles” in RN android apps. Unbundles are created by invoking the packager with the `unbundle` command rather than `bundle`.

The code detects usage of an “unbundle” by checking for the existence of a specific asset.

Reviewed By: astreet

Differential Revision: D2739596

fb-gh-sync-id: d0813c003fe0fa7b47798b970f56707079bfa5d7

ReactAndroid/src/main/jni/react/Android.mk

@@ -9,6 +9,7 @@ LOCAL_SRC_FILES := \
   JSCExecutor.cpp \
   JSCHelpers.cpp \
   JSCWebWorker.cpp \
+  JSModulesUnbundle.cpp \
   MethodCall.cpp \
   Value.cpp \
 

ReactAndroid/src/main/jni/react/Bridge.cpp

@@ -27,6 +27,13 @@ public:
     m_jsExecutor->executeApplicationScript(script, sourceURL);
   }
 
+  void loadApplicationUnbundle(
+      JSModulesUnbundle&& unbundle,
+      const std::string& startupCode,
+      const std::string& sourceURL) {
+    m_jsExecutor->loadApplicationUnbundle(std::move(unbundle), startupCode, sourceURL);
+  }
+
   void flush() {
     auto returnedJSON = m_jsExecutor->flush();
     m_callback(parseMethodCalls(returnedJSON), true /* = isEndOfBatch */);
@@ -95,6 +102,13 @@ void Bridge::executeApplicationScript(const std::string& script, const std::stri
   m_threadState->executeApplicationScript(script, sourceURL);
 }
 
+void Bridge::loadApplicationUnbundle(
+    JSModulesUnbundle&& unbundle,
+    const std::string& startupCode,
+    const std::string& sourceURL) {
+  m_threadState->loadApplicationUnbundle(std::move(unbundle), startupCode, sourceURL);
+}
+
 void Bridge::flush() {
   if (*m_destroyed) {
     return;

ReactAndroid/src/main/jni/react/Bridge.h

@@ -11,6 +11,7 @@
 #include "Value.h"
 #include "Executor.h"
 #include "MethodCall.h"
+#include "JSModulesUnbundle.h"
 
 namespace folly {
 
@@ -45,7 +46,20 @@ public:
    */
   void invokeCallback(const double callbackId, const folly::dynamic& args);
 
+  /**
+   * Starts the JS application from an "bundle", i.e. a JavaScript file that
+   * contains code for all modules and a runtime that resolves and
+   * executes modules.
+   */
   void executeApplicationScript(const std::string& script, const std::string& sourceURL);
+  /**
+   * Starts the JS application from an "unbundle", i.e. a backend that stores
+   * and injects each module as individual file.
+   */
+  void loadApplicationUnbundle(
+    JSModulesUnbundle&& unbundle,
+    const std::string& startupCode,
+    const std::string& sourceURL);
   void setGlobalVariable(const std::string& propName, const std::string& jsonValue);
   bool supportsProfiling();
   void startProfiler(const std::string& title);

ReactAndroid/src/main/jni/react/Executor.h

@@ -6,6 +6,7 @@
 #include <vector>
 #include <memory>
 #include <jni/Countable.h>
+#include "JSModulesUnbundle.h"
 
 namespace folly {
 
@@ -35,6 +36,14 @@ public:
     const std::string& script,
     const std::string& sourceURL) = 0;
 
+  /**
+   * Add an application "unbundle" file
+   */
+  virtual void loadApplicationUnbundle(
+    JSModulesUnbundle&& bundle,
+    const std::string& startupCode,
+    const std::string& sourceURL) = 0;
+
   /**
    * Executes BatchedBridge.flushedQueue in JS to get the next queue of changes.
    */

ReactAndroid/src/main/jni/react/JSCExecutor.cpp

@@ -14,6 +14,7 @@
 #include "Value.h"
 #include "jni/JMessageQueueThread.h"
 #include "jni/OnLoad.h"
+#include <react/JSCHelpers.h>
 
 #ifdef WITH_JSC_EXTRA_TRACING
 #include <react/JSCTracing.h>
@@ -152,12 +153,24 @@ void JSCExecutor::executeApplicationScript(
   env->DeleteLocalRef(startStringMarker);
   env->DeleteLocalRef(endStringMarker);
 
-  String jsSourceURL(sourceURL.c_str());
   #ifdef WITH_FBSYSTRACE
   FbSystraceSection s(TRACE_TAG_REACT_CXX_BRIDGE, "JSCExecutor::executeApplicationScript",
     "sourceURL", sourceURL);
   #endif
-  evaluateScript(m_context, jsScript, jsSourceURL);
+  evaluateScript(m_context, jsScript, String(sourceURL.c_str()));
+}
+
+void JSCExecutor::loadApplicationUnbundle(
+    JSModulesUnbundle&& unbundle,
+    const std::string& startupCode,
+    const std::string& sourceURL) {
+
+  m_unbundle = std::move(unbundle);
+  if (!m_isUnbundleInitialized) {
+    m_isUnbundleInitialized = true;
+    installGlobalFunction(m_context, "nativeRequire", nativeRequire);
+  }
+  executeApplicationScript(startupCode, sourceURL);
 }
 
 std::string JSCExecutor::flush() {
@@ -238,6 +251,13 @@ void JSCExecutor::flushQueueImmediate(std::string queueJSON) {
   m_flushImmediateCallback(queueJSON, false);
 }
 
+void JSCExecutor::loadModule(uint32_t moduleId) {
+  auto module = m_unbundle.getModule(moduleId);
+  auto sourceUrl = String::createExpectingAscii(module.name);
+  auto source = String::createExpectingAscii(module.code);
+  evaluateScript(m_context, source, sourceUrl);
+}
+
 // WebWorker impl
 
 JSGlobalContextRef JSCExecutor::getContext() {
@@ -287,12 +307,55 @@ void JSCExecutor::terminateWebWorker(int workerId) {
   m_webWorkerJSObjs.erase(workerId);
 }
 
+// Native JS hooks
+
+static JSValueRef makeInvalidModuleIdJSCException(
+    JSContextRef ctx,
+    const JSValueRef id,
+    JSValueRef *exception) {
+  std::string message = "Received invalid module ID: ";
+  message += String::adopt(JSValueToStringCopy(ctx, id, exception)).str();
+  return makeJSCException(ctx, message.c_str());
+}
+
+JSValueRef JSCExecutor::nativeRequire(
+  JSContextRef ctx,
+  JSObjectRef function,
+  JSObjectRef thisObject,
+  size_t argumentCount,
+  const JSValueRef arguments[],
+  JSValueRef *exception) {
+
+  if (argumentCount != 1) {
+    *exception = makeJSCException(ctx, "Got wrong number of args");
+    return JSValueMakeUndefined(ctx);
+  }
+
+  JSCExecutor *executor;
+  try {
+    executor = s_globalContextRefToJSCExecutor.at(JSContextGetGlobalContext(ctx));
+  } catch (std::out_of_range& e) {
+    *exception = makeJSCException(ctx, "Global JS context didn't map to a valid executor");
+    return JSValueMakeUndefined(ctx);
+  }
+
+  double moduleId = JSValueToNumber(ctx, arguments[0], exception);
+  if (moduleId <= (double) UINT32_MAX && moduleId >= 0.0) {
+    try {
+      executor->loadModule(moduleId);
+    } catch (JSModulesUnbundle::ModuleNotFound&) {
+      *exception = makeInvalidModuleIdJSCException(ctx, arguments[0], exception);
+    }
+  } else {
+    *exception = makeInvalidModuleIdJSCException(ctx, arguments[0], exception);
+  }
+  return JSValueMakeUndefined(ctx);
+}
+
 static JSValueRef createErrorString(JSContextRef ctx, const char *msg) {
   return JSValueMakeString(ctx, String(msg));
 }
 
-// Native JS hooks
-
 static JSValueRef nativeFlushQueueImmediate(
     JSContextRef ctx,
     JSObjectRef function,

ReactAndroid/src/main/jni/react/JSCExecutor.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <memory>
 #include <unordered_map>
 #include <JavaScriptCore/JSContextRef.h>
@@ -30,6 +31,10 @@ public:
   virtual void executeApplicationScript(
     const std::string& script,
     const std::string& sourceURL) override;
+  virtual void loadApplicationUnbundle(
+    JSModulesUnbundle&& unbundle,
+    const std::string& startupCode,
+    const std::string& sourceURL) override;
   virtual std::string flush() override;
   virtual std::string callFunction(
     const double moduleId,
@@ -59,10 +64,13 @@ private:
   std::unordered_map<int, JSCWebWorker> m_webWorkers;
   std::unordered_map<int, Object> m_webWorkerJSObjs;
   std::shared_ptr<JMessageQueueThread> m_messageQueueThread;
+  JSModulesUnbundle m_unbundle;
+  bool m_isUnbundleInitialized = false;
 
   int addWebWorker(const std::string& script, JSValueRef workerRef);
   void postMessageToWebWorker(int worker, JSValueRef message, JSValueRef *exn);
   void terminateWebWorker(int worker);
+  void loadModule(uint32_t moduleId);
 
   static JSValueRef nativeStartWorker(
       JSContextRef ctx,
@@ -85,6 +93,13 @@ private:
       size_t argumentCount,
       const JSValueRef arguments[],
       JSValueRef *exception);
+  static JSValueRef nativeRequire(
+    JSContextRef ctx,
+    JSObjectRef function,
+    JSObjectRef thisObject,
+    size_t argumentCount,
+    const JSValueRef arguments[],
+    JSValueRef *exception);
 };
 
 } }

ReactAndroid/src/main/jni/react/JSCHelpers.cpp

@@ -41,13 +41,14 @@ JSValueRef evaluateScript(JSContextRef context, JSStringRef script, JSStringRef
     FBLOGE("Got JS Exception: %s", exceptionText.c_str());
     auto line = exception.asObject().getProperty("line");
 
-    std::ostringstream lineInfo;
+    std::ostringstream locationInfo;
+    std::string file = String::adopt(source).str();
+    locationInfo << "(" << (file.length() ? file : "<unknown file>");
     if (line != nullptr && line.isNumber()) {
-      lineInfo << " (line " << line.asInteger() << " in the generated bundle)";
-    } else {
-      lineInfo << " (no line info)";
+      locationInfo << ":" << line.asInteger();
     }
-    throwJSExecutionException("%s%s", exceptionText.c_str(), lineInfo.str().c_str());
+    locationInfo << ")";
+    throwJSExecutionException("%s %s", exceptionText.c_str(), locationInfo.str().c_str());
   }
   return result;
 }

ReactAndroid/src/main/jni/react/JSCHelpers.h

@@ -8,8 +8,6 @@
 
 #define throwJSExecutionException(...) jni::throwNewJavaException("com/facebook/react/bridge/JSExecutionException", __VA_ARGS__)
 
-#define throwJSExecutionException(...) jni::throwNewJavaException("com/facebook/react/bridge/JSExecutionException", __VA_ARGS__)
-
 namespace facebook {
 namespace react {
 

ReactAndroid/src/main/jni/react/JSModulesUnbundle.cpp

@@ -0,0 +1,94 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#include "JSModulesUnbundle.h"
+
+#include <cstdint>
+#include <fb/assert.h>
+#include <libgen.h>
+#include <memory>
+#include <sstream>
+#include <sys/endian.h>
+#include <utility>
+
+using magic_number_t = uint32_t;
+const magic_number_t MAGIC_FILE_HEADER = 0xFB0BD1E5;
+const std::string MAGIC_FILE_NAME = "UNBUNDLE";
+
+namespace facebook {
+namespace react {
+
+using asset_ptr =
+  std::unique_ptr<AAsset, std::function<decltype(AAsset_close)>>;
+
+static std::string jsModulesDir(const std::string& entryFile) {
+  std::string dir = dirname(entryFile.c_str());
+
+  // android's asset manager does not work with paths that start with a dot
+  return dir == "." ? "js-modules/" : dir + "/js-modules/";
+}
+
+static asset_ptr openAsset(
+    AAssetManager *manager,
+    const std::string& fileName,
+    int mode = AASSET_MODE_STREAMING) {
+  return asset_ptr(
+    AAssetManager_open(manager, fileName.c_str(), mode),
+    AAsset_close);
+}
+
+JSModulesUnbundle::JSModulesUnbundle(AAssetManager *assetManager, const std::string& entryFile) :
+  m_assetManager(assetManager),
+  m_moduleDirectory(jsModulesDir(entryFile)) {}
+
+JSModulesUnbundle::JSModulesUnbundle(JSModulesUnbundle&& other) noexcept {
+  *this = std::move(other);
+}
+
+JSModulesUnbundle& JSModulesUnbundle::operator= (JSModulesUnbundle&& other) noexcept {
+  std::swap(m_assetManager, other.m_assetManager);
+  std::swap(m_moduleDirectory, other.m_moduleDirectory);
+
+  return *this;
+}
+
+bool JSModulesUnbundle::isUnbundle(
+    AAssetManager *assetManager,
+    const std::string& assetName) {
+  if (!assetManager) {
+    return false;
+  }
+
+  auto magicFileName = jsModulesDir(assetName) + MAGIC_FILE_NAME;
+  auto asset = openAsset(assetManager, magicFileName.c_str());
+  if (asset == nullptr) {
+    return false;
+  }
+
+  magic_number_t fileHeader = 0;
+  AAsset_read(asset.get(), &fileHeader, sizeof(fileHeader));
+  return fileHeader == htole32(MAGIC_FILE_HEADER);
+}
+
+JSModulesUnbundle::Module JSModulesUnbundle::getModule(uint32_t moduleId) const {
+  // can be nullptr for default constructor.
+  FBASSERTMSGF(m_assetManager != nullptr, "Unbundle has not been initialized with an asset manager");
+
+  std::ostringstream sourceUrlBuilder;
+  sourceUrlBuilder << moduleId << ".js";
+  auto sourceUrl = sourceUrlBuilder.str();
+
+  auto fileName = m_moduleDirectory + sourceUrl;
+  auto asset = openAsset(m_assetManager, fileName, AASSET_MODE_BUFFER);
+
+  const char *buffer = nullptr;
+  if (asset != nullptr) {
+    buffer = static_cast<const char *>(AAsset_getBuffer(asset.get()));
+  }
+  if (buffer == nullptr) {
+    throw ModuleNotFound("Module not found: " + sourceUrl);
+  }
+  return {sourceUrl, std::string(buffer, AAsset_getLength(asset.get()))};
+}
+
+}
+}

ReactAndroid/src/main/jni/react/JSModulesUnbundle.h

@@ -0,0 +1,48 @@
+// Copyright 2004-present Facebook. All Rights Reserved.
+
+#pragma once
+
+#include <android/asset_manager.h>
+#include <cstdint>
+#include <fb/noncopyable.h>
+#include <string>
+#include <stdexcept>
+
+namespace facebook {
+namespace react {
+
+class JSModulesUnbundle : noncopyable {
+  /**
+   * Represents the set of JavaScript modules that the application consists of.
+   * The source code of each module can be retrieved by module ID.
+   * This implementation reads modules as single file from the assets of an apk.
+   *
+   * The class is non-copyable because copying instances might involve copying
+   * several megabytes of memory.
+   */
+public:
+  class ModuleNotFound : public std::out_of_range {
+    using std::out_of_range::out_of_range;
+  };
+  struct Module {
+    std::string name;
+    std::string code;
+  };
+
+  JSModulesUnbundle() = default;
+  JSModulesUnbundle(AAssetManager *assetManager, const std::string& entryFile);
+  JSModulesUnbundle(JSModulesUnbundle&& other) noexcept;
+  JSModulesUnbundle& operator= (JSModulesUnbundle&& other) noexcept;
+
+  static bool isUnbundle(
+    AAssetManager *assetManager,
+    const std::string& assetName);
+  Module getModule(uint32_t moduleId) const;
+
+private:
+  AAssetManager *m_assetManager = nullptr;
+  std::string m_moduleDirectory;
+};
+
+}
+}

ReactAndroid/src/main/jni/react/Value.cpp

@@ -3,7 +3,6 @@
 #include "Value.h"
 
 #include <jni/fbjni.h>
-#include <fb/log.h>
 
 #include "JSCHelpers.h"
 

ReactAndroid/src/main/jni/react/jni/JSLoader.cpp

@@ -2,7 +2,6 @@
 
 #include "JSLoader.h"
 
-#include <android/asset_manager.h>
 #include <android/asset_manager_jni.h>
 #include <jni/Environment.h>
 #include <fstream>
@@ -10,7 +9,6 @@
 #include <streambuf>
 #include <string>
 #include <fb/log.h>
-
 #ifdef WITH_FBSYSTRACE
 #include <fbsystrace.h>
 using fbsystrace::FbSystraceSection;
@@ -29,19 +27,16 @@ std::string loadScriptFromAssets(std::string assetName) {
     gApplicationHolderClass,
     gGetApplicationMethod);
   jobject assetManager = env->CallObjectMethod(application, gGetAssetManagerMethod);
-  return loadScriptFromAssets(env, assetManager, assetName);
+  return loadScriptFromAssets(AAssetManager_fromJava(env, assetManager), assetName);
 }
 
 std::string loadScriptFromAssets(
-    JNIEnv *env,
-    jobject assetManager,
+    AAssetManager *manager,
     std::string assetName) {
   #ifdef WITH_FBSYSTRACE
   FbSystraceSection s(TRACE_TAG_REACT_CXX_BRIDGE, "reactbridge_jni_loadScriptFromAssets",
     "assetName", assetName);
   #endif
-
-  auto manager = AAssetManager_fromJava(env, assetManager);
   if (manager) {
     auto asset = AAssetManager_open(
       manager,
@@ -90,7 +85,7 @@ void registerJSLoaderNatives() {
   jclass applicationHolderClass = env->FindClass("com/facebook/react/common/ApplicationHolder");
   gApplicationHolderClass = (jclass)env->NewGlobalRef(applicationHolderClass);
   gGetApplicationMethod = env->GetStaticMethodID(applicationHolderClass, "getApplication", "()Landroid/app/Application;");
-  
+
   jclass appClass = env->FindClass("android/app/Application");
   gGetAssetManagerMethod = env->GetMethodID(appClass, "getAssets", "()Landroid/content/res/AssetManager;");
 }

ReactAndroid/src/main/jni/react/jni/JSLoader.h

@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <android/asset_manager.h>
 #include <string>
 #include <jni.h>
 
@@ -17,7 +18,7 @@ std::string loadScriptFromAssets(std::string assetName);
 /**
  * Helper method for loading JS script from android asset
  */
-std::string loadScriptFromAssets(JNIEnv *env, jobject assetManager, std::string assetName);
+std::string loadScriptFromAssets(AAssetManager *assetManager, std::string assetName);
 
 /**
  * Helper method for loading JS script from a file

ReactAndroid/src/main/jni/react/jni/OnLoad.cpp

@@ -1,5 +1,6 @@
 // Copyright 2004-present Facebook. All Rights Reserved.
 
+#include <android/asset_manager_jni.h>
 #include <android/input.h>
 #include <fb/log.h>
 #include <folly/json.h>
@@ -13,6 +14,7 @@
 #include <react/Bridge.h>
 #include <react/Executor.h>
 #include <react/JSCExecutor.h>
+#include <react/JSModulesUnbundle.h>
 #include "JNativeRunnable.h"
 #include "JSLoader.h"
 #include "ReadableNativeArray.h"
@@ -643,15 +645,33 @@ static void executeApplicationScript(
   }
 }
 
+static void loadApplicationUnbundle(
+    const RefPtr<Bridge>& bridge,
+    AAssetManager *assetManager,
+    const std::string& startupCode,
+    const std::string& startupFileName) {
+
+  try {
+    // Load the application unbundle and collect/dispatch any native calls that might have occured
+    bridge->loadApplicationUnbundle(
+      JSModulesUnbundle(assetManager, startupFileName),
+      startupCode,
+      startupFileName);
+    bridge->flush();
+  } catch (...) {
+    translatePendingCppExceptionToJavaException();
+  }
+}
+
 static void loadScriptFromAssets(JNIEnv* env, jobject obj, jobject assetManager,
                                  jstring assetName) {
   jclass markerClass = env->FindClass("com/facebook/react/bridge/ReactMarker");
-
+  auto manager = AAssetManager_fromJava(env, assetManager);
   auto bridge = extractRefPtr<Bridge>(env, obj);
   auto assetNameStr = fromJString(env, assetName);
 
   env->CallStaticVoidMethod(markerClass, gLogMarkerMethod, env->NewStringUTF("loadScriptFromAssets_start"));
-  auto script = react::loadScriptFromAssets(env, assetManager, assetNameStr);
+  auto script = react::loadScriptFromAssets(manager, assetNameStr);
   #ifdef WITH_FBSYSTRACE
   FbSystraceSection s(TRACE_TAG_REACT_CXX_BRIDGE, "reactbridge_jni_"
     "executeApplicationScript",
@@ -659,7 +679,11 @@ static void loadScriptFromAssets(JNIEnv* env, jobject obj, jobject assetManager,
   #endif
 
   env->CallStaticVoidMethod(markerClass, gLogMarkerMethod, env->NewStringUTF("loadScriptFromAssets_read"));
-  executeApplicationScript(bridge, script, assetNameStr);
+  if (JSModulesUnbundle::isUnbundle(manager, assetNameStr)) {
+    loadApplicationUnbundle(bridge, manager, script, assetNameStr);
+  } else {
+    executeApplicationScript(bridge, script, assetNameStr);
+  }
   if (env->ExceptionCheck()) {
     return;
   }

ReactAndroid/src/main/jni/react/jni/ProxyExecutor.cpp

@@ -2,6 +2,7 @@
 
 #include "ProxyExecutor.h"
 
+#include <fb/assert.h>
 #include <jni/Environment.h>
 #include <jni/LocalReference.h>
 #include <jni/LocalString.h>
@@ -50,6 +51,11 @@ void ProxyExecutor::executeApplicationScript(
     jni::make_jstring(sourceURL).get());
 }
 
+void ProxyExecutor::loadApplicationUnbundle(JSModulesUnbundle&&, const std::string&, const std::string&) {
+  jni::throwNewJavaException(
+    "java/lang/UnsupportedOperationException",
+    "Loading application unbundles is not supported for proxy executors");
+}
 
 std::string ProxyExecutor::flush() {
   return executeJSCallWithProxy(m_executor.get(), "flushedQueue", std::vector<folly::dynamic>());

ReactAndroid/src/main/jni/react/jni/ProxyExecutor.h

@@ -32,6 +32,10 @@ public:
   virtual void executeApplicationScript(
     const std::string& script,
     const std::string& sourceURL) override;
+  virtual void loadApplicationUnbundle(
+    JSModulesUnbundle&& bundle,
+    const std::string& startupCode,
+    const std::string& sourceURL) override;
   virtual std::string flush() override;
   virtual std::string callFunction(
     const double moduleId,

local-cli/bundle/output/unbundle/as-indexed-file.js

@@ -76,7 +76,7 @@ function uInt32Buffer(n) {
 
 function buildModuleTable(buffers) {
   // table format:
-  //  - table_length: uint_32 length of all table entries in bytes
+  //  - table_length: uint_32 length of all table entries in bytes + the table length itself
   //  - entries: entry...
   //
   // entry:

packager/react-packager/src/Resolver/polyfills/require-unbundle.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const {ErrorUtils, __nativeRequire} = global;
+const {ErrorUtils, nativeRequire} = global;
 global.require = require;
 global.__d = define;
 
@@ -32,7 +32,7 @@ function guardedLoadModule(moduleId, module) {
 
 function loadModuleImplementation(moduleId, module) {
   if (!module) {
-    __nativeRequire(moduleId);
+    nativeRequire(moduleId);
     module = modules[moduleId];
   }