feat(android): integrating Google Vision's text recognition

android/src/main/java/org/reactnative/camera/CameraViewManager.java

@@ -19,7 +19,8 @@ public class CameraViewManager extends ViewGroupManager<RNCameraView> {
     EVENT_ON_MOUNT_ERROR("onMountError"),
     EVENT_ON_BAR_CODE_READ("onBarCodeRead"),
     EVENT_ON_FACES_DETECTED("onFacesDetected"),
-    EVENT_ON_FACE_DETECTION_ERROR("onFaceDetectionError");
+    EVENT_ON_FACE_DETECTION_ERROR("onFaceDetectionError"),
+    EVENT_ON_TEXT_RECOGNIZED("onTextRecognized");
 
     private final String mName;
 
@@ -138,4 +139,9 @@ public class CameraViewManager extends ViewGroupManager<RNCameraView> {
   public void setFaceDetectionClassifications(RNCameraView view, int classifications) {
     view.setFaceDetectionClassifications(classifications);
   }
+
+  @ReactProp(name = "textRecognizerEnabled")
+  public void setTextRecognizing(RNCameraView view, boolean textRecognizerEnabled) {
+    view.setShouldRecognizeText(textRecognizerEnabled);
+  }
 }

android/src/main/java/org/reactnative/camera/RNCameraView.java

@@ -17,6 +17,9 @@ import com.facebook.react.bridge.WritableMap;
 import com.facebook.react.uimanager.ThemedReactContext;
 import com.google.android.cameraview.CameraView;
 import com.google.android.gms.vision.face.Face;
+import com.google.android.gms.vision.text.Text;
+import com.google.android.gms.vision.text.TextBlock;
+import com.google.android.gms.vision.text.TextRecognizer;
 import com.google.zxing.BarcodeFormat;
 import com.google.zxing.DecodeHintType;
 import com.google.zxing.MultiFormatReader;
@@ -27,6 +30,8 @@ import org.reactnative.camera.tasks.BarCodeScannerAsyncTaskDelegate;
 import org.reactnative.camera.tasks.FaceDetectorAsyncTask;
 import org.reactnative.camera.tasks.FaceDetectorAsyncTaskDelegate;
 import org.reactnative.camera.tasks.ResolveTakenPictureAsyncTask;
+import org.reactnative.camera.tasks.TextRecognizerAsyncTask;
+import org.reactnative.camera.tasks.TextRecognizerAsyncTaskDelegate;
 import org.reactnative.camera.utils.ImageDimensions;
 import org.reactnative.camera.utils.RNFileUtils;
 import org.reactnative.facedetector.RNFaceDetector;
@@ -41,7 +46,8 @@ import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
-public class RNCameraView extends CameraView implements LifecycleEventListener, BarCodeScannerAsyncTaskDelegate, FaceDetectorAsyncTaskDelegate {
+public class RNCameraView extends CameraView implements LifecycleEventListener, BarCodeScannerAsyncTaskDelegate, FaceDetectorAsyncTaskDelegate,
+    TextRecognizerAsyncTaskDelegate {
   private ThemedReactContext mThemedReactContext;
   private Queue<Promise> mPictureTakenPromises = new ConcurrentLinkedQueue<>();
   private Map<Promise, ReadableMap> mPictureTakenOptions = new ConcurrentHashMap<>();
@@ -55,12 +61,15 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
   // Concurrency lock for scanners to avoid flooding the runtime
   public volatile boolean barCodeScannerTaskLock = false;
   public volatile boolean faceDetectorTaskLock = false;
+  public volatile boolean textRecognizerTaskLock = false;
 
   // Scanning-related properties
   private final MultiFormatReader mMultiFormatReader = new MultiFormatReader();
   private final RNFaceDetector mFaceDetector;
+  private final TextRecognizer mTextRecognizer;
   private boolean mShouldDetectFaces = false;
   private boolean mShouldScanBarCodes = false;
+  private boolean mShouldRecognizeText = false;
   private int mFaceDetectorMode = RNFaceDetector.FAST_MODE;
   private int mFaceDetectionLandmarks = RNFaceDetector.NO_LANDMARKS;
   private int mFaceDetectionClassifications = RNFaceDetector.NO_CLASSIFICATIONS;
@@ -71,6 +80,7 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
     mThemedReactContext = themedReactContext;
     mFaceDetector = new RNFaceDetector(themedReactContext);
     setupFaceDetector();
+    mTextRecognizer = new TextRecognizer.Builder(themedReactContext).build();
     themedReactContext.addLifecycleEventListener(this);
 
     addCallback(new Callback() {
@@ -121,6 +131,12 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
           FaceDetectorAsyncTaskDelegate delegate = (FaceDetectorAsyncTaskDelegate) cameraView;
           new FaceDetectorAsyncTask(delegate, mFaceDetector, data, width, height, correctRotation).execute();
         }
+
+        if (mShouldRecognizeText && !textRecognizerTaskLock && cameraView instanceof TextRecognizerAsyncTaskDelegate) {
+          textRecognizerTaskLock = true;
+          TextRecognizerAsyncTaskDelegate delegate = (TextRecognizerAsyncTaskDelegate) cameraView;
+          new TextRecognizerAsyncTask(delegate, mTextRecognizer, data, width, height, correctRotation).execute();
+        }
       }
     });
   }
@@ -145,7 +161,7 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
   @Override
   public void onViewAdded(View child) {
     if (this.getView() == child || this.getView() == null) return;
-    // remove and readd view to make sure it is in the back.
+    // remove and read view to make sure it is in the back.
     // @TODO figure out why there was a z order issue in the first place and fix accordingly.
     this.removeView(this.getView());
     this.addView(this.getView(), 0);
@@ -210,7 +226,7 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
 
   public void setShouldScanBarCodes(boolean shouldScanBarCodes) {
     this.mShouldScanBarCodes = shouldScanBarCodes;
-    setScanning(mShouldDetectFaces || mShouldScanBarCodes);
+    setScanning(mShouldDetectFaces || mShouldScanBarCodes || mShouldRecognizeText);
   }
 
   public void onBarCodeRead(Result barCode) {
@@ -260,7 +276,7 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
 
   public void setShouldDetectFaces(boolean shouldDetectFaces) {
     this.mShouldDetectFaces = shouldDetectFaces;
-    setScanning(mShouldDetectFaces || mShouldScanBarCodes);
+    setScanning(mShouldDetectFaces || mShouldScanBarCodes || mShouldRecognizeText);
   }
 
   public void onFacesDetected(SparseArray<Face> facesReported, int sourceWidth, int sourceHeight, int sourceRotation) {
@@ -287,6 +303,28 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
     faceDetectorTaskLock = false;
   }
 
+  public void setShouldRecognizeText(boolean shouldRecognizeText) {
+    this.mShouldRecognizeText = shouldRecognizeText;
+    setScanning(mShouldDetectFaces || mShouldScanBarCodes || mShouldRecognizeText);
+  }
+
+  @Override
+  public void onTextRecognized(SparseArray<TextBlock> textBlocks, int sourceWidth, int sourceHeight, int sourceRotation) {
+    if (!mShouldRecognizeText) {
+      return;
+    }
+
+    SparseArray<TextBlock> textBlocksDetected = textBlocks == null ? new SparseArray<TextBlock>() : textBlocks;
+    ImageDimensions dimensions = new ImageDimensions(sourceWidth, sourceHeight, sourceRotation, getFacing());
+
+    RNCameraViewHelper.emitTextRecognizedEvent(this, textBlocksDetected, dimensions);
+  }
+
+  @Override
+  public void onTextRecognizerTaskCompleted() {
+    textRecognizerTaskLock = false;
+  }
+
   @Override
   public void onHostResume() {
     if (hasCameraPermissions()) {

android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java

@@ -16,6 +16,7 @@ import com.facebook.react.bridge.WritableMap;
 import com.facebook.react.uimanager.UIManagerModule;
 import com.google.android.cameraview.CameraView;
 import com.google.android.gms.vision.face.Face;
+import com.google.android.gms.vision.text.TextBlock;
 import com.google.zxing.Result;
 
 import org.reactnative.camera.events.BarCodeReadEvent;
@@ -23,6 +24,7 @@ import org.reactnative.camera.events.CameraMountErrorEvent;
 import org.reactnative.camera.events.CameraReadyEvent;
 import org.reactnative.camera.events.FaceDetectionErrorEvent;
 import org.reactnative.camera.events.FacesDetectedEvent;
+import org.reactnative.camera.events.TextRecognizedEvent;
 import org.reactnative.camera.utils.ImageDimensions;
 import org.reactnative.facedetector.RNFaceDetector;
 
@@ -217,6 +219,29 @@ public class RNCameraViewHelper {
     reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event);
   }
 
+  // Text recognition event
+
+  public static void emitTextRecognizedEvent(
+      ViewGroup view,
+      SparseArray<TextBlock> textBlocks,
+      ImageDimensions dimensions) {
+    float density = view.getResources().getDisplayMetrics().density;
+
+    double scaleX = (double) view.getWidth() / (dimensions.getWidth() * density);
+    double scaleY = (double) view.getHeight() / (dimensions.getHeight() * density);
+
+    TextRecognizedEvent event = TextRecognizedEvent.obtain(
+        view.getId(),
+        textBlocks,
+        dimensions,
+        scaleX,
+        scaleY
+    );
+
+    ReactContext reactContext = (ReactContext) view.getContext();
+    reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event);
+  }
+
   // Utilities
 
   public static int getCorrectCameraRotation(int rotation, int facing) {

android/src/main/java/org/reactnative/camera/events/TextRecognizedEvent.java

@@ -0,0 +1,157 @@
+package org.reactnative.camera.events;
+
+import android.support.v4.util.Pools;
+import android.util.SparseArray;
+
+import com.facebook.react.bridge.Arguments;
+import com.facebook.react.bridge.ReadableArray;
+import com.facebook.react.bridge.ReadableMap;
+import com.facebook.react.bridge.WritableArray;
+import com.facebook.react.bridge.WritableMap;
+import com.facebook.react.uimanager.events.Event;
+import com.facebook.react.uimanager.events.RCTEventEmitter;
+import com.google.android.cameraview.CameraView;
+import com.google.android.gms.vision.text.Line;
+import com.google.android.gms.vision.text.Text;
+import com.google.android.gms.vision.text.TextBlock;
+import org.reactnative.camera.CameraViewManager;
+import org.reactnative.camera.utils.ImageDimensions;
+import org.reactnative.facedetector.FaceDetectorUtils;
+
+
+public class TextRecognizedEvent extends Event<TextRecognizedEvent> {
+
+  private static final Pools.SynchronizedPool<TextRecognizedEvent> EVENTS_POOL =
+      new Pools.SynchronizedPool<>(3);
+
+
+  private double mScaleX;
+  private double mScaleY;
+  private SparseArray<TextBlock> mTextBlocks;
+  private ImageDimensions mImageDimensions;
+
+  private TextRecognizedEvent() {}
+
+  public static TextRecognizedEvent obtain(
+      int viewTag,
+      SparseArray<TextBlock> textBlocks,
+      ImageDimensions dimensions,
+      double scaleX,
+      double scaleY) {
+    TextRecognizedEvent event = EVENTS_POOL.acquire();
+    if (event == null) {
+      event = new TextRecognizedEvent();
+    }
+    event.init(viewTag, textBlocks, dimensions, scaleX, scaleY);
+    return event;
+  }
+
+  private void init(
+      int viewTag,
+      SparseArray<TextBlock> textBlocks,
+      ImageDimensions dimensions,
+      double scaleX,
+      double scaleY) {
+    super.init(viewTag);
+    mTextBlocks = textBlocks;
+    mImageDimensions = dimensions;
+    mScaleX = scaleX;
+    mScaleY = scaleY;
+  }
+
+  @Override
+  public String getEventName() {
+    return CameraViewManager.Events.EVENT_ON_TEXT_RECOGNIZED.toString();
+  }
+
+  @Override
+  public void dispatch(RCTEventEmitter rctEventEmitter) {
+    rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData());
+  }
+
+  private WritableMap serializeEventData() {
+    WritableArray textBlocksList = Arguments.createArray();
+    for (int i = 0; i < mTextBlocks.size(); ++i) {
+      TextBlock textBlock = mTextBlocks.valueAt(i);
+      WritableMap serializedTextBlock = serializeText(textBlock);
+      if (mImageDimensions.getFacing() == CameraView.FACING_FRONT) {
+        serializedTextBlock = rotateTextX(serializedTextBlock);
+      }
+      textBlocksList.pushMap(serializedTextBlock);
+    }
+
+    WritableMap event = Arguments.createMap();
+    event.putString("type", "textBlock");
+    event.putArray("textBlocks", textBlocksList);
+    event.putInt("target", getViewTag());
+    return event;
+  }
+
+  private WritableMap serializeText(Text text) {
+    WritableMap encodedText = Arguments.createMap();
+
+    WritableArray components = Arguments.createArray();
+    for (Text component : text.getComponents()) {
+      components.pushMap(serializeText(component));
+    }
+    encodedText.putArray("components", components);
+
+    encodedText.putString("value", text.getValue());
+
+    WritableMap origin = Arguments.createMap();
+    origin.putDouble("x", text.getBoundingBox().left * this.mScaleX);
+    origin.putDouble("y", text.getBoundingBox().top * this.mScaleY);
+
+    WritableMap size = Arguments.createMap();
+    size.putDouble("width", text.getBoundingBox().width() * this.mScaleX);
+    size.putDouble("height", text.getBoundingBox().width() * this.mScaleY);
+
+    WritableMap bounds = Arguments.createMap();
+    bounds.putMap("origin", origin);
+    bounds.putMap("size", size);
+
+    encodedText.putMap("bounds", bounds);
+
+    String type_;
+    if (text instanceof TextBlock) {
+      type_ = "block";
+    } else if (text instanceof Line) {
+      type_ = "line";
+    } else /*if (text instanceof Element)*/ {
+      type_ = "element";
+    }
+    encodedText.putString("type", type_);
+
+    return encodedText;
+  }
+
+  private WritableMap rotateTextX(WritableMap text) {
+    ReadableMap faceBounds = text.getMap("bounds");
+
+    ReadableMap oldOrigin = faceBounds.getMap("origin");
+    WritableMap mirroredOrigin = FaceDetectorUtils.positionMirroredHorizontally(
+        oldOrigin, mImageDimensions.getWidth(), mScaleX);
+
+    double translateX = -faceBounds.getMap("size").getDouble("width");
+    WritableMap translatedMirroredOrigin = FaceDetectorUtils.positionTranslatedHorizontally(mirroredOrigin, translateX);
+
+    WritableMap newBounds = Arguments.createMap();
+    newBounds.merge(faceBounds);
+    newBounds.putMap("origin", translatedMirroredOrigin);
+
+    text.putMap("bounds", newBounds);
+
+    ReadableArray oldComponents = text.getArray("components");
+    WritableArray newComponents = Arguments.createArray();
+    for (int i = 0; i < oldComponents.size(); ++i) {
+      WritableMap component = Arguments.createMap();
+      component.merge(oldComponents.getMap(i));
+      rotateTextX(component);
+      newComponents.pushMap(component);
+    }
+    text.putArray("components", newComponents);
+
+    return text;
+  }
+
+}

android/src/main/java/org/reactnative/camera/tasks/TextRecognizerAsyncTask.java

@@ -0,0 +1,55 @@
+package org.reactnative.camera.tasks;
+
+import android.util.SparseArray;
+
+import com.google.android.gms.vision.text.TextBlock;
+import com.google.android.gms.vision.text.TextRecognizer;
+import org.reactnative.facedetector.RNFrame;
+import org.reactnative.facedetector.RNFrameFactory;
+
+
+public class TextRecognizerAsyncTask extends android.os.AsyncTask<Void, Void, SparseArray<TextBlock>> {
+
+  private TextRecognizerAsyncTaskDelegate mDelegate;
+  private TextRecognizer mTextRecognizer;
+  private byte[] mImageData;
+  private int mWidth;
+  private int mHeight;
+  private int mRotation;
+
+  public TextRecognizerAsyncTask(
+      TextRecognizerAsyncTaskDelegate delegate,
+      TextRecognizer textRecognizer,
+      byte[] imageData,
+      int width,
+      int height,
+      int rotation
+  ) {
+    mDelegate = delegate;
+    mTextRecognizer = textRecognizer;
+    mImageData = imageData;
+    mWidth = width;
+    mHeight = height;
+    mRotation = rotation;
+  }
+
+  @Override
+  protected SparseArray<TextBlock> doInBackground(Void... ignored) {
+    if (isCancelled() || mDelegate == null || mTextRecognizer == null || !mTextRecognizer.isOperational()) {
+      return null;
+    }
+
+    RNFrame frame = RNFrameFactory.buildFrame(mImageData, mWidth, mHeight, mRotation);
+    return mTextRecognizer.detect(frame.getFrame());
+  }
+
+  @Override
+  protected void onPostExecute(SparseArray<TextBlock> textBlocks) {
+    super.onPostExecute(textBlocks);
+
+    if (textBlocks != null) {
+      mDelegate.onTextRecognized(textBlocks, mWidth, mHeight, mRotation);
+    }
+    mDelegate.onTextRecognizerTaskCompleted();
+  }
+}

android/src/main/java/org/reactnative/camera/tasks/TextRecognizerAsyncTaskDelegate.java

@@ -0,0 +1,10 @@
+package org.reactnative.camera.tasks;
+
+import android.util.SparseArray;
+
+import com.google.android.gms.vision.text.TextBlock;
+
+public interface TextRecognizerAsyncTaskDelegate {
+  void onTextRecognized(SparseArray<TextBlock> textBlocks, int sourceWidth, int sourceHeight, int sourceRotation);
+  void onTextRecognizerTaskCompleted();
+}

src/RNCamera.js

@@ -58,6 +58,7 @@ type PropsType = ViewPropTypes & {
   autoFocus?: string | boolean | number,
   faceDetectionClassifications?: number,
   onFacesDetected?: ({ faces: Array<TrackedFaceFeature> }) => void,
+  onTextRecognized?: Function,
   captureAudio?: boolean,
   useCamera2Api?: boolean,
 };
@@ -122,6 +123,7 @@ export default class Camera extends React.Component<PropsType> {
     onCameraReady: PropTypes.func,
     onBarCodeRead: PropTypes.func,
     onFacesDetected: PropTypes.func,
+    onTextRecognized: PropTypes.func,
     faceDetectionMode: PropTypes.number,
     faceDetectionLandmarks: PropTypes.number,
     faceDetectionClassifications: PropTypes.number,
@@ -295,6 +297,7 @@ export default class Camera extends React.Component<PropsType> {
           onCameraReady={this._onCameraReady}
           onBarCodeRead={this._onObjectDetected(this.props.onBarCodeRead)}
           onFacesDetected={this._onObjectDetected(this.props.onFacesDetected)}
+          onTextRecognized={this._onObjectDetected(this.props.onTextRecognized)}
         />
       );
     } else if (!this.state.isAuthorizationChecked) {
@@ -315,8 +318,13 @@ export default class Camera extends React.Component<PropsType> {
       newProps.faceDetectorEnabled = true;
     }
 
+    if (props.onTextRecognized) {
+      newProps.textRecognizerEnabled = true;
+    }
+
     if (Platform.OS === 'ios') {
       delete newProps.ratio;
+      delete newProps.textRecognizerEnabled;
     }
 
     return newProps;
@@ -340,6 +348,7 @@ const RNCamera = requireNativeComponent('RNCamera', Camera, {
     accessibilityLiveRegion: true,
     barCodeScannerEnabled: true,
     faceDetectorEnabled: true,
+    textRecognizerEnabled: true,
     importantForAccessibility: true,
     onBarCodeRead: true,
     onCameraReady: true,