diff --git a/src/caffe2-ios/caffe2-ios.xcodeproj/project.pbxproj b/src/caffe2-ios/caffe2-ios.xcodeproj/project.pbxproj index 1817242..051e4e4 100644 --- a/src/caffe2-ios/caffe2-ios.xcodeproj/project.pbxproj +++ b/src/caffe2-ios/caffe2-ios.xcodeproj/project.pbxproj @@ -7,11 +7,13 @@ objects = { /* Begin PBXBuildFile section */ - 7C03C1A71EBE4E9900BCA7E6 /* libCAFFE2_NNPACK.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C03C1A51EBE4E9900BCA7E6 /* libCAFFE2_NNPACK.a */; }; - 7C03C1A81EBE4E9900BCA7E6 /* libCAFFE2_PTHREADPOOL.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C03C1A61EBE4E9900BCA7E6 /* libCAFFE2_PTHREADPOOL.a */; }; - 7C03C1AB1EBE4EAB00BCA7E6 /* libprotobuf-lite.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C03C1A91EBE4EAB00BCA7E6 /* libprotobuf-lite.a */; }; - 7C03C1AC1EBE4EAB00BCA7E6 /* libprotobuf.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C03C1AA1EBE4EAB00BCA7E6 /* libprotobuf.a */; }; - 7C03C1B01EBE844A00BCA7E6 /* libCaffe2_CPU.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C03C1AF1EBE844A00BCA7E6 /* libCaffe2_CPU.a */; }; + 7C06346D1F513A120060D94E /* libprotobuf-lite.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C06346B1F513A120060D94E /* libprotobuf-lite.a */; }; + 7C06346E1F513A120060D94E /* libprotobuf.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C06346C1F513A120060D94E /* libprotobuf.a */; }; + 7C0634711F513A250060D94E /* libCAFFE2_NNPACK.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C06346F1F513A250060D94E /* libCAFFE2_NNPACK.a */; }; + 7C0634721F513A250060D94E /* libCAFFE2_PTHREADPOOL.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C0634701F513A250060D94E /* libCAFFE2_PTHREADPOOL.a */; }; + 7C0634741F513A370060D94E /* libCaffe2_CPU.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C0634731F513A370060D94E /* libCaffe2_CPU.a */; }; + 7C0634791F52B85D0060D94E /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C0634751F52B5780060D94E /* Metal.framework */; }; + 7C06347A1F52B8650060D94E /* MetalPerformanceShaders.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7C0634771F52B57E0060D94E /* MetalPerformanceShaders.framework */; }; 7C5702811EBC47C4002F6227 /* tinyYoloInit.pb in Resources */ = {isa = PBXBuildFile; fileRef = 7C57027F1EBC47C4002F6227 /* tinyYoloInit.pb */; }; 7C5702821EBC47C4002F6227 /* tinyYoloPredict.pb in Resources */ = {isa = PBXBuildFile; fileRef = 7C5702801EBC47C4002F6227 /* tinyYoloPredict.pb */; }; 7C8E442C1EB329FC00BA6D76 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C8E442B1EB329FC00BA6D76 /* AppDelegate.swift */; }; @@ -39,11 +41,13 @@ /* End PBXBuildFile section */ /* Begin PBXFileReference section */ - 7C03C1A51EBE4E9900BCA7E6 /* libCAFFE2_NNPACK.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libCAFFE2_NNPACK.a; sourceTree = ""; }; - 7C03C1A61EBE4E9900BCA7E6 /* libCAFFE2_PTHREADPOOL.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libCAFFE2_PTHREADPOOL.a; sourceTree = ""; }; - 7C03C1A91EBE4EAB00BCA7E6 /* libprotobuf-lite.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libprotobuf-lite.a"; sourceTree = ""; }; - 7C03C1AA1EBE4EAB00BCA7E6 /* libprotobuf.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libprotobuf.a; sourceTree = ""; }; - 7C03C1AF1EBE844A00BCA7E6 /* libCaffe2_CPU.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libCaffe2_CPU.a; sourceTree = ""; }; + 7C06346B1F513A120060D94E /* libprotobuf-lite.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libprotobuf-lite.a"; sourceTree = ""; }; + 7C06346C1F513A120060D94E /* libprotobuf.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libprotobuf.a; sourceTree = ""; }; + 7C06346F1F513A250060D94E /* libCAFFE2_NNPACK.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libCAFFE2_NNPACK.a; sourceTree = ""; }; + 7C0634701F513A250060D94E /* libCAFFE2_PTHREADPOOL.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libCAFFE2_PTHREADPOOL.a; sourceTree = ""; }; + 7C0634731F513A370060D94E /* libCaffe2_CPU.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libCaffe2_CPU.a; sourceTree = ""; }; + 7C0634751F52B5780060D94E /* Metal.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Metal.framework; path = System/Library/Frameworks/Metal.framework; sourceTree = SDKROOT; }; + 7C0634771F52B57E0060D94E /* MetalPerformanceShaders.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = MetalPerformanceShaders.framework; path = System/Library/Frameworks/MetalPerformanceShaders.framework; sourceTree = SDKROOT; }; 7C57027F1EBC47C4002F6227 /* tinyYoloInit.pb */ = {isa = PBXFileReference; lastKnownFileType = file; path = tinyYoloInit.pb; sourceTree = ""; }; 7C5702801EBC47C4002F6227 /* tinyYoloPredict.pb */ = {isa = PBXFileReference; lastKnownFileType = file; path = tinyYoloPredict.pb; sourceTree = ""; }; 7C8E44281EB329FC00BA6D76 /* caffe2-ios.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "caffe2-ios.app"; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -82,17 +86,19 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 7C06347A1F52B8650060D94E /* MetalPerformanceShaders.framework in Frameworks */, + 7C0634791F52B85D0060D94E /* Metal.framework in Frameworks */, 7C9DDAA71EB5627D0037EC52 /* AVFoundation.framework in Frameworks */, + 7C0634721F513A250060D94E /* libCAFFE2_PTHREADPOOL.a in Frameworks */, 7CE985571EB41F9B00CA112F /* UIKit.framework in Frameworks */, 7CE985551EB41F9400CA112F /* Foundation.framework in Frameworks */, 7CE985531EB41F8C00CA112F /* Accelerate.framework in Frameworks */, - 7C03C1AC1EBE4EAB00BCA7E6 /* libprotobuf.a in Frameworks */, - 7C03C1B01EBE844A00BCA7E6 /* libCaffe2_CPU.a in Frameworks */, + 7C0634741F513A370060D94E /* libCaffe2_CPU.a in Frameworks */, + 7C0634711F513A250060D94E /* libCAFFE2_NNPACK.a in Frameworks */, + 7C06346E1F513A120060D94E /* libprotobuf.a in Frameworks */, + 7C06346D1F513A120060D94E /* libprotobuf-lite.a in Frameworks */, 7CE9854E1EB41DB700CA112F /* libstdc++.tbd in Frameworks */, 7C8E44521EB32EAF00BA6D76 /* opencv2.framework in Frameworks */, - 7C03C1A81EBE4E9900BCA7E6 /* libCAFFE2_PTHREADPOOL.a in Frameworks */, - 7C03C1A71EBE4E9900BCA7E6 /* libCAFFE2_NNPACK.a in Frameworks */, - 7C03C1AB1EBE4EAB00BCA7E6 /* libprotobuf-lite.a in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -157,6 +163,8 @@ 7CE9854C1EB41DB700CA112F /* Frameworks */ = { isa = PBXGroup; children = ( + 7C0634771F52B57E0060D94E /* MetalPerformanceShaders.framework */, + 7C0634751F52B5780060D94E /* Metal.framework */, 7C9DDAA61EB5627D0037EC52 /* AVFoundation.framework */, 7CE985561EB41F9B00CA112F /* UIKit.framework */, 7CE985541EB41F9400CA112F /* Foundation.framework */, @@ -169,11 +177,11 @@ 7CE9854F1EB41DFA00CA112F /* caffe2 */ = { isa = PBXGroup; children = ( - 7C03C1AF1EBE844A00BCA7E6 /* libCaffe2_CPU.a */, - 7C03C1A91EBE4EAB00BCA7E6 /* libprotobuf-lite.a */, - 7C03C1AA1EBE4EAB00BCA7E6 /* libprotobuf.a */, - 7C03C1A51EBE4E9900BCA7E6 /* libCAFFE2_NNPACK.a */, - 7C03C1A61EBE4E9900BCA7E6 /* libCAFFE2_PTHREADPOOL.a */, + 7C0634731F513A370060D94E /* libCaffe2_CPU.a */, + 7C06346F1F513A250060D94E /* libCAFFE2_NNPACK.a */, + 7C0634701F513A250060D94E /* libCAFFE2_PTHREADPOOL.a */, + 7C06346B1F513A120060D94E /* libprotobuf-lite.a */, + 7C06346C1F513A120060D94E /* libprotobuf.a */, ); name = caffe2; sourceTree = ""; @@ -455,6 +463,7 @@ "$(SRCROOT)/../caffe2/third_party/eigen", ); INFOPLIST_FILE = "caffe2-ios/Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 10.3; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; LIBRARY_SEARCH_PATHS = ( "$(inherited)", @@ -492,6 +501,7 @@ "$(SRCROOT)/../caffe2/third_party/eigen", ); INFOPLIST_FILE = "caffe2-ios/Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 10.3; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; LIBRARY_SEARCH_PATHS = ( "$(inherited)", diff --git a/src/caffe2-ios/caffe2-ios/Caffe2.h b/src/caffe2-ios/caffe2-ios/Caffe2.h index df6c3ba..fbf046b 100644 --- a/src/caffe2-ios/caffe2-ios/Caffe2.h +++ b/src/caffe2-ios/caffe2-ios/Caffe2.h @@ -15,11 +15,13 @@ - (null_unspecified instancetype)init UNAVAILABLE_ATTRIBUTE; +- (nullable)loadModelAndTryConvertToMPSCNN; + - (null_unspecified instancetype) init:(nonnull NSString*)initNetFilename predict:(nonnull NSString*)predictNetFilename error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(init(initNetNamed:predictNetNamed:)); -- (nullable NSArray*) predict:(nonnull UIImage*) image -NS_SWIFT_NAME(prediction(regarding:)); +- (nullable NSArray*) predictWithGPU:(nonnull UIImage*) image +NS_SWIFT_NAME(predictionGPU(regarding:)); - (nullable) reloadModel:(nonnull NSString*)initNetFilename predict:(nonnull NSString*)predictNetFilename error:(NSError * _Nullable * _Nullable)error NS_SWIFT_NAME(reloadModel(initNetNamed:predictNetNamed:)); diff --git a/src/caffe2-ios/caffe2-ios/Caffe2.mm b/src/caffe2-ios/caffe2-ios/Caffe2.mm index a7c53f7..e6b1c18 100644 --- a/src/caffe2-ios/caffe2-ios/Caffe2.mm +++ b/src/caffe2-ios/caffe2-ios/Caffe2.mm @@ -9,8 +9,11 @@ #import #import "Caffe2.h" #include "caffe2/core/predictor.h" +#include "caffe2/core/flags.h" #include "caffe2/utils/proto_utils.h" +#include "caffe2/contrib/ios/mpscnn/mpscnn.h" +CAFFE2_DECLARE_bool(caffe2_force_shared_col_buffer); void ReadProtoIntoNet(std::string fname, caffe2::NetDef* net) { int file = open(fname.c_str(), O_RDONLY); @@ -80,6 +83,7 @@ CGContextRef CreateRGBABitmapContext (CGImageRef inImage) @interface Caffe2(){ caffe2::NetDef _initNet; caffe2::NetDef _predictNet; + caffe2::NetDef _mpscnnPredictNet; caffe2::Predictor *_predictor; } @@ -113,14 +117,12 @@ - (instancetype) init:(nonnull NSString*)initNetFilename predict:(nonnull NSStri ReadProtoIntoNet(initNetPath.UTF8String, &_initNet); ReadProtoIntoNet(predictNetPath.UTF8String, &_predictNet); - _predictNet.set_name("PredictNet"); - _predictor = new caffe2::Predictor(_initNet, _predictNet); + [self loadModelAndTryConvertToMPSCNN]; } return self; } -(void) reloadModel:(nonnull NSString*)initNetFilename predict:(nonnull NSString*)predictNetFilename error:(NSError **)error { - if(self){ delete _predictor; @@ -133,8 +135,26 @@ -(void) reloadModel:(nonnull NSString*)initNetFilename predict:(nonnull NSString ReadProtoIntoNet(initNetPath.UTF8String, &_initNet); ReadProtoIntoNet(predictNetPath.UTF8String, &_predictNet); - _predictNet.set_name("PredictNet"); - _predictor = new caffe2::Predictor(_initNet, _predictNet); + [self loadModelAndTryConvertToMPSCNN]; + + + + } +} + +// iOS A9 GPU is required + +-(void)loadModelAndTryConvertToMPSCNN{ + _predictNet.set_name("PredictNet"); + _mpscnnPredictNet.set_name("MPSCNNPredictNet"); + + if (caffe2::tryConvertToMPSCNN(_initNet, _predictNet, &_mpscnnPredictNet)){ + caffe2::dumpDef(_mpscnnPredictNet); + _predictor = new caffe2::Predictor(_initNet, _mpscnnPredictNet); + NSLog(@"[Caffe2 Wrapper] MPSCNN Converting successful!"); + } else { + _predictor = new caffe2::Predictor(_initNet,_predictNet); + NSLog(@"[Caffe2 Wrapper] MPSCNN Converting failed!"); } } @@ -144,8 +164,7 @@ -(void)loadDownloadedModel:(nonnull NSString*)initNetFilePath predict:(nonnull N ReadProtoIntoNet(initNetFilePath.UTF8String, &_initNet); ReadProtoIntoNet(predictNetFilePath.UTF8String, &_predictNet); - _predictNet.set_name("PredictNet"); - _predictor = new caffe2::Predictor(_initNet, _predictNet); + [self loadModelAndTryConvertToMPSCNN]; } } @@ -154,10 +173,20 @@ -(void)dealloc { google::protobuf::ShutdownProtobufLibrary(); } -- (nullable NSArray*) predict:(nonnull UIImage*) image{ - NSMutableArray* result = nil; - caffe2::Predictor::TensorVector output_vec; + +/* + Code below Refers to: + https://github.com/caffe2/caffe2/blob/master/caffe2/contrib/ios/ios_caffe_predictor.cc + */ +- (nullable NSArray*) predictWithGPU:(nonnull UIImage*) image{ + caffe2::FLAGS_caffe2_force_shared_col_buffer = true; + caffe2::ThreadPool* threadpool = _predictor->ws()->GetThreadPool(); + if (threadpool != nullptr){ + threadpool->setMinWorkSize(std::numeric_limits::max()); + } + + NSMutableArray* result = nil; if (self.busyWithInference) { return nil; } else { @@ -165,28 +194,21 @@ -(void)dealloc { } CGImageRef inImage = image.CGImage; - // Create the bitmap context - // We do this to ensure correct color space layout CGContextRef cgctx = CreateRGBABitmapContext(inImage); if (cgctx == NULL){ return nil; } - // Get image width, height. We'll use the entire image. size_t w = CGImageGetWidth(inImage); size_t h = CGImageGetHeight(inImage); CGRect rect = {{0,0},{static_cast(w),static_cast(h)}}; - // Draw the image to the bitmap context. Once we draw, the memory - // allocated for the context for rendering will then contain the - // raw image data in the specified color space. CGContextDrawImage(cgctx, rect, inImage); void *data = CGBitmapContextGetData (cgctx); if (_predictor && data) { UInt8* pixels = (UInt8*) data; caffe2::TensorCPU input; - // Reasonable dimensions to feed the predictor. const int predHeight = (int)CGSizeEqualToSize(self.imageInputDimensions, CGSizeZero) ? int(h) : self.imageInputDimensions.height; const int predWidth = (int)CGSizeEqualToSize(self.imageInputDimensions, CGSizeZero) ? int(w) : self.imageInputDimensions.width; const int crops = 1; @@ -196,12 +218,12 @@ -(void)dealloc { const float wscale = ((float)w) / predWidth; const float scale = std::min(hscale, wscale); std::vector inputPlanar(crops * channels * predHeight * predWidth); - // Scale down the input to a reasonable predictor size. + for (auto i = 0; i < predHeight; ++i) { const int _i = (int) (scale * i); for (auto j = 0; j < predWidth; ++j) { const int _j = (int) (scale * j); - // The input is of the form RGBA, we only need the RGB part. + float red = (float) pixels[(_i * w + _j) * 4 + 0]; float green = (float) pixels[(_i * w + _j) * 4 + 1]; float blue = (float) pixels[(_i * w + _j) * 4 + 2]; @@ -212,15 +234,17 @@ -(void)dealloc { } } + input.Resize(std::vector({crops, channels, predHeight, predWidth})); input.ShareExternalPointer(inputPlanar.data()); caffe2::Predictor::TensorVector input_vec{&input}; + caffe2::Predictor::TensorVector output_vec; + _predictor->run(input_vec, &output_vec); if (output_vec.capacity() > 0) { for (auto output : output_vec) { - // currently only one dimensional output supported result = [NSMutableArray arrayWithCapacity:output_vec.size()]; for (auto i = 0; i < output->size(); ++i) { result[i] = @(output->template data()[i]); @@ -232,7 +256,7 @@ -(void)dealloc { self.busyWithInference = false; } - // When finished, release the context/ data + CGContextRelease(cgctx); if (data) { free(data); diff --git a/src/caffe2-ios/caffe2-ios/libCAFFE2_NNPACK.a b/src/caffe2-ios/caffe2-ios/libCAFFE2_NNPACK.a deleted file mode 100644 index 3b6ff7b..0000000 Binary files a/src/caffe2-ios/caffe2-ios/libCAFFE2_NNPACK.a and /dev/null differ diff --git a/src/caffe2-ios/caffe2-ios/libCAFFE2_PTHREADPOOL.a b/src/caffe2-ios/caffe2-ios/libCAFFE2_PTHREADPOOL.a deleted file mode 100644 index cd7deb0..0000000 Binary files a/src/caffe2-ios/caffe2-ios/libCAFFE2_PTHREADPOOL.a and /dev/null differ diff --git a/src/caffe2-ios/caffe2-ios/libCaffe2_CPU.a b/src/caffe2-ios/caffe2-ios/libCaffe2_CPU.a deleted file mode 100644 index cae8f04..0000000 Binary files a/src/caffe2-ios/caffe2-ios/libCaffe2_CPU.a and /dev/null differ diff --git a/src/caffe2-ios/caffe2-ios/libprotobuf-lite.a b/src/caffe2-ios/caffe2-ios/libprotobuf-lite.a deleted file mode 100644 index f1ca7ac..0000000 Binary files a/src/caffe2-ios/caffe2-ios/libprotobuf-lite.a and /dev/null differ diff --git a/src/caffe2-ios/caffe2-ios/libprotobuf.a b/src/caffe2-ios/caffe2-ios/libprotobuf.a deleted file mode 100644 index 51b2b18..0000000 Binary files a/src/caffe2-ios/caffe2-ios/libprotobuf.a and /dev/null differ diff --git a/src/caffe2-ios/caffe2-ios/realTimeDetectorVC.swift b/src/caffe2-ios/caffe2-ios/realTimeDetectorVC.swift index 9403d2e..215fd4d 100644 --- a/src/caffe2-ios/caffe2-ios/realTimeDetectorVC.swift +++ b/src/caffe2-ios/caffe2-ios/realTimeDetectorVC.swift @@ -81,7 +81,7 @@ class realTimeDetectorVC: UIViewController, AVCaptureVideoDataOutputSampleBuffer func classifier(img: UIImage){ let start = CACurrentMediaTime() - if let predictedResult = caffe.prediction(regarding: img){ + if let predictedResult = caffe.predictionGPU(regarding: img){ switch modelPicked { case "squeezeNet": let sorted = predictedResult.map{$0.floatValue}.enumerated().sorted(by: {$0.element > $1.element})[0...10] diff --git a/src/caffe2-ios/caffe2-ios/staticDetectorVC.swift b/src/caffe2-ios/caffe2-ios/staticDetectorVC.swift index 38ff107..4f928e6 100644 --- a/src/caffe2-ios/caffe2-ios/staticDetectorVC.swift +++ b/src/caffe2-ios/caffe2-ios/staticDetectorVC.swift @@ -94,7 +94,7 @@ class staticDetectorVC: UIViewController, UIImagePickerControllerDelegate, UINav let start = CACurrentMediaTime() self.imageDisplayer.image = image let resizedImage = resizeImage(image: image, newWidth: CGFloat(500)) - if let result = caffe.prediction(regarding: resizedImage!){ + if let result = caffe.predictionGPU(regarding: resizedImage!){ let end = CACurrentMediaTime() self.elapse = "\(end - start) seconds" diff --git a/src/setup.sh b/src/setup.sh index 6f4f806..64f49e9 100644 --- a/src/setup.sh +++ b/src/setup.sh @@ -5,7 +5,7 @@ echo "[Step2] git cloning Caffe2" git clone --recursive https://github.com/caffe2/caffe2.git echo "[Step2 - b] checkout to a stable version tags/v0.7.0-163-gebc17cc8" -cd caffe2 && git checkout tags/v0.7.0-163-gebc17cc8 +cd caffe2 && git checkout tags/v0.8.1-157-g5b696c12 echo "[Step3] build ios" ./scripts/build_ios.sh