status-im/react-native-camera
1
0
Fork 0
mirror of https://github.com/status-im/react-native-camera.git synced 2026-08-31 05:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improved face detector and callback

Browse Source
This commit is contained in:
Malte Peters
2018-02-08 15:27:07 +01:00
parent 4019eb28d4
commit fa913bd4cc
5 changed files with 1588 additions and 26318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ios/RN/OpenCVProcessor.mm
+108 -146
View File
@@ -2,11 +2,19 @@
#import <opencv2/opencv.hpp>
#import <opencv2/objdetect.hpp>
@implementation OpenCVProcessor
@implementation OpenCVProcessor{
BOOL saveDemoFrame;
int processedFrames;
}
- (id) init {
NSString *path = [[NSBundle mainBundle] pathForResource:@"haarcascade_frontalface_alt.xml"
saveDemoFrame = false;
processedFrames = 0;
NSString *path = [[NSBundle mainBundle] pathForResource:@"lbpcascade_frontalface_improved.xml"
ofType:nil];
std::string cascade_path = (char *)[path UTF8String];
if (!cascade.load(cascade_path)) {
NSLog(@"Couldn't load haar cascade file.");
@@ -26,174 +34,128 @@
# pragma mark - OpenCV-Processing
#ifdef __cplusplus
- (void)saveImageToDisk:(Mat&)image;
{
NSLog(@"----------------SAVE IMAGE-----------------");
saveDemoFrame = false;
NSData *data = [NSData dataWithBytes:image.data length:image.elemSize()*image.total()];
CGColorSpaceRef colorSpace;
if (image.elemSize() == 1) {
colorSpace = CGColorSpaceCreateDeviceGray();
} else {
colorSpace = CGColorSpaceCreateDeviceRGB();
}
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)data);
// Creating CGImage from cv::Mat
CGImageRef imageRef = CGImageCreate(image.cols, //width
image.rows, //height
8, //bits per component
8 * image.elemSize(), //bits per pixel
image.step[0], //bytesPerRow
colorSpace, //colorspace
kCGImageAlphaNone|kCGBitmapByteOrderDefault,// bitmap info
provider, //CGDataProviderRef
NULL, //decode
false, //should interpolate
kCGRenderingIntentDefault //intent
);
// Getting UIImage from CGImage
UIImage *finalImage = [UIImage imageWithCGImage:imageRef];
CGImageRelease(imageRef);
CGDataProviderRelease(provider);
CGColorSpaceRelease(colorSpace);
UIImageWriteToSavedPhotosAlbum(finalImage, nil, nil, nil);
}
- (void)processImage:(Mat&)image;
{
cv::Mat grayMat;
cv::cvtColor(image, grayMat, CV_BGR2GRAY);
//cv::equalizeHist(image, image);
cv::equalizeHist(grayMat, grayMat);
// rotate image according to device-rotation
UIDeviceOrientation interfaceOrientation = [[UIDevice currentDevice] orientation];
if (interfaceOrientation == UIDeviceOrientationPortrait) {
transpose(image, image);
flip(image, image,1);
} else if (interfaceOrientation == UIDeviceOrientationPortraitUpsideDown) {
transpose(image, image);
flip(image, image,0);
} else if (interfaceOrientation == UIDeviceOrientationLandscapeLeft) {
flip(image, image,-1);
}
cv::resize(image, image, cv::Size(0,0), 360./(float)image.cols, 360./(float)image.cols, cv::INTER_CUBIC);
if(saveDemoFrame){
[self saveImageToDisk:image];
}
objects.clear();
cascade.detectMultiScale(grayMat, objects,
4.6, 1,
cascade.detectMultiScale(image,
objects,
2.0,
3,
CV_HAAR_SCALE_IMAGE,
cv::Size(40, 40));
cv::Size(30, 30));
for(size_t i = 0; i < objects.size(); ++i) {
[delegate onFacesDetected:[NSArray new]];
if(objects.size() > 0){
NSMutableArray *faces = [[NSMutableArray alloc] initWithCapacity:objects.size()];
for( int i = 0; i < objects.size(); i++ )
{
cv::Rect face = objects[i];
id objects[] = { @(face.x), @(face.y), @(face.width), @(face.height) };
id keys[] = { @"x", @"y", @"width", @"height" };
NSUInteger count = sizeof(objects) / sizeof(id);
NSDictionary *faceDescriptor = [NSDictionary dictionaryWithObjects:objects
forKeys:keys count:count];
[faces addObject:faceDescriptor];
}
[delegate onFacesDetected:faces];
}
}
- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection
{
(void)captureOutput;
(void)connection;
// convert from Core Media to Core Video
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
CVPixelBufferLockBaseAddress(imageBuffer, 0);
void* bufferAddress;
size_t width;
size_t height;
size_t bytesPerRow;
CGColorSpaceRef colorSpace;
CGContextRef context;
int format_opencv;
OSType format = CVPixelBufferGetPixelFormatType(imageBuffer);
if (format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
// https://github.com/opencv/opencv/blob/master/modules/videoio/src/cap_ios_video_camera.mm
if(processedFrames % 10 == 0){
(void)captureOutput;
(void)connection;
format_opencv = CV_8UC1;
// convert from Core Media to Core Video
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
CVPixelBufferLockBaseAddress(imageBuffer, 0);
void* bufferAddress;
size_t width;
size_t height;
size_t bytesPerRow;
int format_opencv = CV_8UC1;
bufferAddress = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
width = CVPixelBufferGetWidthOfPlane(imageBuffer, 0);
height = CVPixelBufferGetHeightOfPlane(imageBuffer, 0);
bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
} else { // expect kCVPixelFormatType_32BGRA
// delegate image processing to the delegate
cv::Mat image((int)height, (int)width, format_opencv, bufferAddress, bytesPerRow);
format_opencv = CV_8UC4;
bufferAddress = CVPixelBufferGetBaseAddress(imageBuffer);
width = CVPixelBufferGetWidth(imageBuffer);
height = CVPixelBufferGetHeight(imageBuffer);
bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
[self processImage:image];
// cleanup
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
}
// delegate image processing to the delegate
cv::Mat image((int)height, (int)width, format_opencv, bufferAddress, bytesPerRow);
CGImage* dstImage;
[self processImage:image];
// check if matrix data pointer or dimensions were changed by the delegate
bool iOSimage = false;
if (height == (size_t)image.rows && width == (size_t)image.cols && format_opencv == image.type() && bufferAddress == image.data && bytesPerRow == image.step) {
iOSimage = true;
}
// (create color space, create graphics context, render buffer)
CGBitmapInfo bitmapInfo;
// basically we decide if it's a grayscale, rgb or rgba image
if (image.channels() == 1) {
colorSpace = CGColorSpaceCreateDeviceGray();
bitmapInfo = kCGImageAlphaNone;
} else if (image.channels() == 3) {
colorSpace = CGColorSpaceCreateDeviceRGB();
bitmapInfo = kCGImageAlphaNone;
if (iOSimage) {
bitmapInfo |= kCGBitmapByteOrder32Little;
} else {
bitmapInfo |= kCGBitmapByteOrder32Big;
}
} else {
colorSpace = CGColorSpaceCreateDeviceRGB();
bitmapInfo = kCGImageAlphaPremultipliedFirst;
if (iOSimage) {
bitmapInfo |= kCGBitmapByteOrder32Little;
} else {
bitmapInfo |= kCGBitmapByteOrder32Big;
}
}
if (iOSimage) {
context = CGBitmapContextCreate(bufferAddress, width, height, 8, bytesPerRow, colorSpace, bitmapInfo);
dstImage = CGBitmapContextCreateImage(context);
CGContextRelease(context);
} else {
NSData *data = [NSData dataWithBytes:image.data length:image.elemSize()*image.total()];
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)data);
// Creating CGImage from cv::Mat
dstImage = CGImageCreate(image.cols, // width
image.rows, // height
8, // bits per component
8 * image.elemSize(), // bits per pixel
image.step, // bytesPerRow
colorSpace, // colorspace
bitmapInfo, // bitmap info
provider, // CGDataProviderRef
NULL, // decode
false, // should interpolate
kCGRenderingIntentDefault // intent
);
CGDataProviderRelease(provider);
}
// render buffer
dispatch_sync(dispatch_get_main_queue(), ^{
// self.customPreviewLayer.contents = (__bridge id)dstImage;
});
// recordingCountDown--;
// if (self.recordVideo == YES && recordingCountDown < 0) {
// lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
// // CMTimeShow(lastSampleTime);
// if (self.recordAssetWriter.status != AVAssetWriterStatusWriting) {
// [self.recordAssetWriter startWriting];
// [self.recordAssetWriter startSessionAtSourceTime:lastSampleTime];
// if (self.recordAssetWriter.status != AVAssetWriterStatusWriting) {
// NSLog(@"[Camera] Recording Error: asset writer status is not writing: %@", self.recordAssetWriter.error);
// return;
// } else {
// NSLog(@"[Camera] Video recording started");
// }
// }
//
// if (self.recordAssetWriterInput.readyForMoreMediaData) {
// CVImageBufferRef pixelBuffer = [self pixelBufferFromCGImage:dstImage];
// if (! [self.recordPixelBufferAdaptor appendPixelBuffer:pixelBuffer
// withPresentationTime:lastSampleTime] ) {
// NSLog(@"Video Writing Error");
// }
// if (pixelBuffer != nullptr)
// CVPixelBufferRelease(pixelBuffer);
// }
//
// }
// cleanup
CGImageRelease(dstImage);
CGColorSpaceRelease(colorSpace);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
processedFrames++;
}
#endif
@end
ios/RN/RNCamera.m
+7 -7
View File
@@ -38,8 +38,8 @@ static NSDictionary *defaultFaceDetectorOptions = nil;
self.bridge = bridge;
self.session = [AVCaptureSession new];
self.sessionQueue = dispatch_queue_create("cameraQueue", DISPATCH_QUEUE_SERIAL);
// self.faceDetectorManager = [self createFaceDetectorManager];
self.openCVProcessor = [OpenCVProcessor new];
// self.faceDetectorManager = [self createFaceDetectorManager];
self.openCVProcessor = [[OpenCVProcessor new] initWithDelegate:self];
#if !(TARGET_IPHONE_SIMULATOR)
self.previewLayer =
[AVCaptureVideoPreviewLayer layerWithSession:self.session];
@@ -368,7 +368,7 @@ static NSDictionary *defaultFaceDetectorOptions = nil;
// At the time of writing AVCaptureMovieFileOutput and AVCaptureVideoDataOutput (> GMVDataOutput)
// cannot coexist on the same AVSession (see: https://stackoverflow.com/a/4986032/1123156).
// We stop face detection here and restart it in when AVCaptureMovieFileOutput finishes recording.
// [_faceDetectorManager stopFaceDetection];
// [_faceDetectorManager stopFaceDetection];
[self setupMovieFileCapture];
}
@@ -431,7 +431,7 @@ static NSDictionary *defaultFaceDetectorOptions = nil;
// create VideoOutput for processing
AVCaptureVideoDataOutput *videoDataOutput = [AVCaptureVideoDataOutput new];
NSDictionary *newSettings = @{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA) };
NSDictionary *newSettings = @{ (NSString *)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) };
videoDataOutput.videoSettings = newSettings;
// discard if the data output queue is blocked (as we process the still image
@@ -453,7 +453,7 @@ static NSDictionary *defaultFaceDetectorOptions = nil;
}
// [_faceDetectorManager maybeStartFaceDetectionOnSession:_session withPreviewLayer:_previewLayer];
// [_faceDetectorManager maybeStartFaceDetectionOnSession:_session withPreviewLayer:_previewLayer];
[self setupOrDisableBarcodeScanner];
__weak RNCamera *weakSelf = self;
@@ -479,7 +479,7 @@ static NSDictionary *defaultFaceDetectorOptions = nil;
return;
#endif
dispatch_async(self.sessionQueue, ^{
// [_faceDetectorManager stopFaceDetection];
// [_faceDetectorManager stopFaceDetection];
[self.previewLayer removeFromSuperlayer];
[self.session commitConfiguration];
[self.session stopRunning];
@@ -734,7 +734,7 @@ static NSDictionary *defaultFaceDetectorOptions = nil;
[self cleanupMovieFileCapture];
// If face detection has been running prior to recording to file
// we reenable it here (see comment in -record).
// [_faceDetectorManager maybeStartFaceDetectionOnSession:_session withPreviewLayer:_previewLayer];
// [_faceDetectorManager maybeStartFaceDetectionOnSession:_session withPreviewLayer:_previewLayer];
if (self.session.sessionPreset != AVCaptureSessionPresetHigh) {
[self updateSessionPreset:AVCaptureSessionPresetHigh];
ios/RN/lbpcascade_frontalface_improved.xml
Executable
+1469
View File
File diff suppressed because it is too large Load Diff
ios/RNCamera.xcodeproj/project.pbxproj
+4 -4
View File
@@ -8,8 +8,8 @@
/* Begin PBXBuildFile section */
001F67882027265A001A21D8 /* opencv2.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 001F67872027265A001A21D8 /* opencv2.framework */; };
001F678A202727FF001A21D8 /* haarcascade_frontalface_alt.xml in CopyFiles */ = {isa = PBXBuildFile; fileRef = 001F6789202727F0001A21D8 /* haarcascade_frontalface_alt.xml */; };
001F681120274350001A21D8 /* OpenCVProcessor.mm in Sources */ = {isa = PBXBuildFile; fileRef = 001F681020274350001A21D8 /* OpenCVProcessor.mm */; };
0036F41F202C958F002EF644 /* lbpcascade_frontalface_improved.xml in CopyFiles */ = {isa = PBXBuildFile; fileRef = 0036F41E202C9563002EF644 /* lbpcascade_frontalface_improved.xml */; };
0314E39D1B661A460092D183 /* CameraFocusSquare.m in Sources */ = {isa = PBXBuildFile; fileRef = 0314E39C1B661A460092D183 /* CameraFocusSquare.m */; };
4107014D1ACB732B00C6AA39 /* RCTCamera.m in Sources */ = {isa = PBXBuildFile; fileRef = 410701481ACB732B00C6AA39 /* RCTCamera.m */; };
4107014E1ACB732B00C6AA39 /* RCTCameraManager.m in Sources */ = {isa = PBXBuildFile; fileRef = 4107014A1ACB732B00C6AA39 /* RCTCameraManager.m */; };
@@ -35,7 +35,7 @@
dstPath = "include/$(PRODUCT_NAME)";
dstSubfolderSpec = 16;
files = (
001F678A202727FF001A21D8 /* haarcascade_frontalface_alt.xml in CopyFiles */,
0036F41F202C958F002EF644 /* lbpcascade_frontalface_improved.xml in CopyFiles */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -43,9 +43,9 @@
/* Begin PBXFileReference section */
001F67872027265A001A21D8 /* opencv2.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = opencv2.framework; path = ../../../../../Downloads/opencv2.framework; sourceTree = "<group>"; };
001F6789202727F0001A21D8 /* haarcascade_frontalface_alt.xml */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xml; path = haarcascade_frontalface_alt.xml; sourceTree = SOURCE_ROOT; };
001F680F2027431C001A21D8 /* OpenCVProcessor.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = OpenCVProcessor.hpp; sourceTree = "<group>"; };
001F681020274350001A21D8 /* OpenCVProcessor.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = OpenCVProcessor.mm; sourceTree = "<group>"; };
0036F41E202C9563002EF644 /* lbpcascade_frontalface_improved.xml */ = {isa = PBXFileReference; lastKnownFileType = text.xml; path = lbpcascade_frontalface_improved.xml; sourceTree = "<group>"; };
0314E39B1B661A0C0092D183 /* CameraFocusSquare.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CameraFocusSquare.h; sourceTree = "<group>"; };
0314E39C1B661A460092D183 /* CameraFocusSquare.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CameraFocusSquare.m; sourceTree = "<group>"; };
4107012F1ACB723B00C6AA39 /* libRNCamera.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libRNCamera.a; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -138,7 +138,7 @@
714166162013E1B600EE9FCC /* RN */ = {
isa = PBXGroup;
children = (
001F6789202727F0001A21D8 /* haarcascade_frontalface_alt.xml */,
0036F41E202C9563002EF644 /* lbpcascade_frontalface_improved.xml */,
71C7FFD42013C824006EB75A /* RNFileSystem.h */,
71C7FFD52013C824006EB75A /* RNFileSystem.m */,
71C7FFD12013C817006EB75A /* RNImageUtils.h */,
ios/haarcascade_frontalface_alt.xml
-26161
View File
File diff suppressed because it is too large Load Diff