Skip to content
This repository was archived by the owner on Apr 4, 2023. It is now read-only.

Commit 22aacd5

Browse files
EddyVerbruggenEddyVerbruggen
committed
Add custom (TensorFlow Lite) models support to the ML Kit feature #702 
1 parent 5fb5a99 commit 22aacd5

12 files changed

Lines changed: 455 additions & 3 deletions

File tree

‎src/mlkit/custommodel/custommodel-common.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import { MLKitCameraView } from "../mlkit-cameraview";
2+
3+
export abstract class MLKitCustomModel extends MLKitCameraView {
4+
static scanResultEvent: string = "scanResult";
5+
}

‎src/mlkit/custommodel/index.android.ts‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
import { ImageSource } from "tns-core-modules/image-source";
2+
import { MLKitOptions } from "../";
3+
import { MLKitCustomModelOptions, MLKitCustomModelResult } from "./";
4+
import { MLKitCustomModel as MLKitCustomModelBase } from "./custommodel-common";
5+
6+
declare const com: any;
7+
8+
export class MLKitCustomModel extends MLKitCustomModelBase {
9+
10+
protected createDetector(): any {
11+
return getInterpreter();
12+
}
13+
14+
protected createSuccessListener(): any {
15+
return new com.google.android.gms.tasks.OnSuccessListener({
16+
onSuccess: labels => {
17+
18+
if (labels.size() === 0) return;
19+
20+
const result = <MLKitCustomModelResult>{
21+
result: []
22+
};
23+
24+
// see https://github.com/firebase/quickstart-android/blob/0f4c86877fc5f771cac95797dffa8bd026dd9dc7/mlkit/app/src/main/java/com/google/firebase/samples/apps/mlkit/textrecognition/TextRecognitionProcessor.java#L62
25+
for (let i = 0; i < labels.size(); i++) {
26+
const label = labels.get(i);
27+
result.result.push({
28+
text: label.getLabel(),
29+
confidence: label.getConfidence()
30+
});
31+
}
32+
33+
this.notify({
34+
eventName: MLKitCustomModel.scanResultEvent,
35+
object: this,
36+
value: result
37+
});
38+
}
39+
});
40+
}
41+
}
42+
43+
function getInterpreter(): any {
44+
const localSource = new com.google.firebase.ml.custom.model.FirebaseLocalModelSource.Builder("my_local_model")
45+
.setAssetFilePath("mobilenet_quant_v1_224.tflite") // Or setFilePath if you downloaded from your host
46+
.build();
47+
com.google.firebase.ml.custom.FirebaseModelManager.getInstance().registerLocalModelSource(localSource);
48+
49+
const options = new com.google.firebase.ml.custom.FirebaseModelOptions.Builder()
50+
// .setCloudModelName("my_cloud_model")
51+
.setLocalModelName("my_local_model")
52+
.build();
53+
return com.google.firebase.ml.custom.FirebaseModelInterpreter.getInstance(options);
54+
}
55+
56+
export function useCustomModel(options: MLKitCustomModelOptions): Promise<MLKitCustomModelResult> {
57+
return new Promise((resolve, reject) => {
58+
try {
59+
const interpreter = getInterpreter();
60+
61+
const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
62+
onSuccess: labels => {
63+
const result = <MLKitCustomModelResult>{
64+
result: []
65+
};
66+
67+
resolve(result);
68+
interpreter.close();
69+
}
70+
});
71+
72+
const onFailureListener = new com.google.android.gms.tasks.OnFailureListener({
73+
onFailure: exception => reject(exception.getMessage())
74+
});
75+
76+
let intArrayIn = Array.create('int', 4);
77+
intArrayIn[0] = 1;
78+
intArrayIn[1] = 640;
79+
intArrayIn[2] = 480;
80+
intArrayIn[3] = 3;
81+
82+
let intArrayOut = Array.create('int', 2);
83+
intArrayOut[0] = 1;
84+
intArrayOut[1] = 1000;
85+
86+
const inputOutputOptions = new com.google.firebase.ml.custom.FirebaseModelInputOutputOptions.Builder()
87+
.setInputFormat(0, com.google.firebase.ml.custom.FirebaseModelDataType.BYTE, intArrayIn)
88+
.setOutputFormat(0, com.google.firebase.ml.custom.FirebaseModelDataType.FLOAT32, intArrayOut)
89+
.build();
90+
91+
// TODO check native example project
92+
const input = null; // getData();
93+
94+
// input = getYourInputData();
95+
const inputs = new com.google.firebase.ml.custom.FirebaseModelInputs.Builder()
96+
.add(input) // add() as many input arrays as your model requires
97+
.build();
98+
99+
// TODO see https://firebase.google.com/docs/ml-kit/android/use-custom-models
100+
interpreter
101+
.run(inputs, inputOutputOptions)
102+
.addOnSuccessListener(onSuccessListener)
103+
.addOnFailureListener(onFailureListener);
104+
105+
} catch (ex) {
106+
console.log("Error in firebase.mlkit.useCustomModel: " + ex);
107+
reject(ex);
108+
}
109+
});
110+
}
111+
112+
function getImage(options: MLKitOptions): any /* com.google.firebase.ml.vision.common.FirebaseVisionImage */ {
113+
const image: android.graphics.Bitmap = options.image instanceof ImageSource ? options.image.android : options.image.imageSource.android;
114+
return com.google.firebase.ml.vision.common.FirebaseVisionImage.fromBitmap(image);
115+
}

‎src/mlkit/custommodel/index.d.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import { MLKitCameraView, MLKitOptions, MLKitResult } from "../index";
2+
3+
export interface MLKitCustomModelResult extends MLKitResult {
4+
result: any;
5+
}
6+
7+
export interface MLKitCustomModelOptions extends MLKitOptions {
8+
}
9+
10+
export declare function useCustomModel(options: MLKitCustomModelOptions): Promise<MLKitCustomModelResult>;
11+
12+
export declare class MLKitCustomModel extends MLKitCameraView {}

‎src/mlkit/custommodel/index.ios.ts‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import { ImageSource } from "tns-core-modules/image-source";
2+
import { MLKitCustomModelOptions, MLKitCustomModelResult } from "./";
3+
import { MLKitCustomModel as MLKitCustomModelBase } from "./custommodel-common";
4+
5+
declare const TNSMLKitCameraView: any;
6+
7+
export class MLKitCustomModel extends MLKitCustomModelBase {
8+
9+
protected createDetector(): any {
10+
return getInterpreter();
11+
}
12+
13+
protected createSuccessListener(): any {
14+
return (outputs: NSArray<FIRModelOutputs>, error: NSError) => {
15+
if (error !== null) {
16+
console.log(error.localizedDescription);
17+
18+
} else if (outputs !== null) {
19+
const result = <MLKitCustomModelResult>{
20+
result: []
21+
};
22+
23+
console.log(">>> outputs: " + outputs);
24+
25+
this.notify({
26+
eventName: MLKitCustomModel.scanResultEvent,
27+
object: this,
28+
value: result
29+
});
30+
}
31+
}
32+
}
33+
34+
protected rotateRecording(): boolean {
35+
return false;
36+
}
37+
}
38+
39+
function getInterpreter(): FIRModelInterpreter {
40+
const fIRModelDownloadConditions = FIRModelDownloadConditions.alloc().initWithWiFiRequiredIdleRequired(false, false);
41+
42+
const fIRCloudModelSource = FIRCloudModelSource.alloc().initWithModelNameEnableModelUpdatesInitialConditionsUpdateConditions(
43+
"my-custom-model",
44+
true,
45+
fIRModelDownloadConditions,
46+
fIRModelDownloadConditions);
47+
48+
// const cloudModelRegistrationSuccess = FIRModelManager.modelManager().registerCloudModelSource(fIRCloudModelSource);
49+
// console.log("cloudModelRegistrationSuccess: " + cloudModelRegistrationSuccess);
50+
51+
loadLocalModel();
52+
53+
const fIRModelOptions = FIRModelOptions.alloc().initWithCloudModelNameLocalModelName(
54+
// "my-custom-model",
55+
null,
56+
"mobilenet");
57+
58+
return FIRModelInterpreter.modelInterpreterWithOptions(fIRModelOptions);
59+
}
60+
61+
function loadLocalModel(): void {
62+
const localModelFilePath = NSBundle.mainBundle.pathForResourceOfType("mobilenet_quant_v1_224", "tflite");
63+
console.log(">>> localModelFilePath: " + localModelFilePath);
64+
65+
const localModelSource = FIRLocalModelSource.alloc().initWithModelNamePath("mobilenet", localModelFilePath);
66+
console.log(">>> localModelSource: " + localModelSource);
67+
68+
const localModelRegistrationSuccess = FIRModelManager.modelManager().registerLocalModelSource(localModelSource);
69+
console.log("localModelRegistrationSuccess: " + localModelRegistrationSuccess);
70+
}
71+
72+
export function useCustomModel(options: MLKitCustomModelOptions): Promise<MLKitCustomModelResult> {
73+
return new Promise((resolve, reject) => {
74+
try {
75+
const modelInterpreter = getInterpreter();
76+
77+
const inputs = FIRModelInputs.new();
78+
const image: UIImage = options.image instanceof ImageSource ? options.image.ios : options.image.imageSource.ios;
79+
80+
// note that there's a LoC in this native function that crashes the app (see the code for details)
81+
const resizedImg = TNSMLKitCameraView.resizeImage(image);
82+
const successAddingInput = inputs.addInputError(resizedImg);
83+
84+
const inputOptions = FIRModelInputOutputOptions.new();
85+
const arrIn = NSMutableArray.new();
86+
arrIn.addObject(1);
87+
arrIn.addObject(image.size.width);
88+
arrIn.addObject(image.size.height);
89+
arrIn.addObject(3);
90+
91+
const arrOut = NSMutableArray.new();
92+
arrOut.addObject(1);
93+
arrOut.addObject(1001);
94+
95+
inputOptions.setInputFormatForIndexTypeDimensionsError(0, FIRModelElementType.UInt8, <any>arrIn);
96+
inputOptions.setOutputFormatForIndexTypeDimensionsError(0, FIRModelElementType.UInt8, <any>arrOut);
97+
98+
modelInterpreter.runWithInputsOptionsCompletion(inputs, inputOptions, (outputs: FIRModelOutputs, error: NSError) => {
99+
console.log(">>> error: " + error);
100+
console.log(">>> outputs: " + outputs);
101+
102+
if (error !== null) {
103+
reject(error.localizedDescription);
104+
105+
} else if (outputs !== null) {
106+
console.log(">>> outputs.count: " + outputs.outputAtIndexError(0));
107+
const result = <MLKitCustomModelResult>{
108+
result: []
109+
};
110+
111+
console.log(">>> outputs: " + outputs);
112+
113+
resolve(result);
114+
}
115+
});
116+
} catch (ex) {
117+
console.log("Error in firebase.mlkit.useCustomModel: " + ex);
118+
reject(ex);
119+
}
120+
});
121+
}

‎src/platforms/ios/TNSMLKitCamera.framework/Headers/TNSMLKitCameraView.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@
1010
@property (nonatomic) UIImageOrientation imageOrientation;
1111

1212
- (nonnull id)initWithCaptureSession:(nonnull AVCaptureSession *)captureSession;
13+
+ (NSArray * _Nullable)resizeImage:(nonnull UIImage *)image;
1314

1415
@end

‎src/platforms/ios/TNSMLKitCamera.framework/Info.plist‎

0 Bytes
Binary file not shown.

‎src/platforms/ios/TNSMLKitCamera.framework/TNSMLKitCamera‎

18.1 KB
Binary file not shown.

‎src/platforms/ios_lib/TNSMLKitCamera/TNSMLKitCamera.xcodeproj/project.pbxproj‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
A32431BD20A5DFBD00C79104 /* TNSMLKitCameraViewDelegate.h in Headers */ = {isa = PBXBuildFile; fileRef = A32431BC20A5DFBD00C79104 /* TNSMLKitCameraViewDelegate.h */; settings = {ATTRIBUTES = (Public, ); }; };
1111
A32431C120A5E25900C79104 /* AVFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A32431C020A5E25900C79104 /* AVFoundation.framework */; };
1212
A32431C320A5E2A200C79104 /* CoreMedia.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A32431C220A5E2A200C79104 /* CoreMedia.framework */; };
13+
A349852320B321E200EBB136 /* UIImage+TNSMLKitCamera.h in Headers */ = {isa = PBXBuildFile; fileRef = A349852220B321E200EBB136 /* UIImage+TNSMLKitCamera.h */; };
14+
A349852520B3220000EBB136 /* UIImage+TNSMLKitCamera.m in Sources */ = {isa = PBXBuildFile; fileRef = A349852420B3220000EBB136 /* UIImage+TNSMLKitCamera.m */; };
1315
A3905B0A1EBCD37D008D0250 /* TNSMLKitCamera.h in Headers */ = {isa = PBXBuildFile; fileRef = A3905B081EBCD37D008D0250 /* TNSMLKitCamera.h */; settings = {ATTRIBUTES = (Public, ); }; };
1416
A3905B111EBCD434008D0250 /* TNSMLKitCameraView.h in Headers */ = {isa = PBXBuildFile; fileRef = A3905B101EBCD434008D0250 /* TNSMLKitCameraView.h */; settings = {ATTRIBUTES = (Public, ); }; };
1517
A3905B131EBCD44B008D0250 /* TNSMLKitCameraView.m in Sources */ = {isa = PBXBuildFile; fileRef = A3905B121EBCD44B008D0250 /* TNSMLKitCameraView.m */; };
@@ -19,6 +21,8 @@
1921
A32431BC20A5DFBD00C79104 /* TNSMLKitCameraViewDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = TNSMLKitCameraViewDelegate.h; sourceTree = "<group>"; };
2022
A32431C020A5E25900C79104 /* AVFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AVFoundation.framework; path = System/Library/Frameworks/AVFoundation.framework; sourceTree = SDKROOT; };
2123
A32431C220A5E2A200C79104 /* CoreMedia.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreMedia.framework; path = System/Library/Frameworks/CoreMedia.framework; sourceTree = SDKROOT; };
24+
A349852220B321E200EBB136 /* UIImage+TNSMLKitCamera.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "UIImage+TNSMLKitCamera.h"; sourceTree = "<group>"; };
25+
A349852420B3220000EBB136 /* UIImage+TNSMLKitCamera.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "UIImage+TNSMLKitCamera.m"; sourceTree = "<group>"; };
2226
A3905B051EBCD37D008D0250 /* TNSMLKitCamera.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = TNSMLKitCamera.framework; sourceTree = BUILT_PRODUCTS_DIR; };
2327
A3905B081EBCD37D008D0250 /* TNSMLKitCamera.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = TNSMLKitCamera.h; sourceTree = "<group>"; };
2428
A3905B091EBCD37D008D0250 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
@@ -64,6 +68,8 @@
6468
A3905B101EBCD434008D0250 /* TNSMLKitCameraView.h */,
6569
A3905B121EBCD44B008D0250 /* TNSMLKitCameraView.m */,
6670
A32431BC20A5DFBD00C79104 /* TNSMLKitCameraViewDelegate.h */,
71+
A349852220B321E200EBB136 /* UIImage+TNSMLKitCamera.h */,
72+
A349852420B3220000EBB136 /* UIImage+TNSMLKitCamera.m */,
6773
);
6874
path = TNSMLKitCamera;
6975
sourceTree = "<group>";
@@ -86,6 +92,7 @@
8692
files = (
8793
A3905B0A1EBCD37D008D0250 /* TNSMLKitCamera.h in Headers */,
8894
A3905B111EBCD434008D0250 /* TNSMLKitCameraView.h in Headers */,
95+
A349852320B321E200EBB136 /* UIImage+TNSMLKitCamera.h in Headers */,
8996
A32431BD20A5DFBD00C79104 /* TNSMLKitCameraViewDelegate.h in Headers */,
9097
);
9198
runOnlyForDeploymentPostprocessing = 0;
@@ -123,6 +130,7 @@
123130
A3905B041EBCD37D008D0250 = {
124131
CreatedOnToolsVersion = 8.3.2;
125132
DevelopmentTeam = 8Q5F6M3TNS;
133+
LastSwiftMigration = 0930;
126134
ProvisioningStyle = Automatic;
127135
};
128136
};
@@ -159,6 +167,7 @@
159167
isa = PBXSourcesBuildPhase;
160168
buildActionMask = 2147483647;
161169
files = (
170+
A349852520B3220000EBB136 /* UIImage+TNSMLKitCamera.m in Sources */,
162171
A3905B131EBCD44B008D0250 /* TNSMLKitCameraView.m in Sources */,
163172
);
164173
runOnlyForDeploymentPostprocessing = 0;
@@ -169,6 +178,7 @@
169178
A3905B0B1EBCD37D008D0250 /* Debug */ = {
170179
isa = XCBuildConfiguration;
171180
buildSettings = {
181+
ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO;
172182
ALWAYS_SEARCH_USER_PATHS = NO;
173183
CLANG_ANALYZER_NONNULL = YES;
174184
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
@@ -212,6 +222,7 @@
212222
MTL_ENABLE_DEBUG_INFO = YES;
213223
ONLY_ACTIVE_ARCH = NO;
214224
SDKROOT = iphoneos;
225+
SWIFT_OBJC_INTERFACE_HEADER_NAME = "TNSMLKitCamera-Swift.h";
215226
TARGETED_DEVICE_FAMILY = "1,2";
216227
VERSIONING_SYSTEM = "apple-generic";
217228
VERSION_INFO_PREFIX = "";
@@ -221,6 +232,7 @@
221232
A3905B0C1EBCD37D008D0250 /* Release */ = {
222233
isa = XCBuildConfiguration;
223234
buildSettings = {
235+
ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO;
224236
ALWAYS_SEARCH_USER_PATHS = NO;
225237
CLANG_ANALYZER_NONNULL = YES;
226238
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
@@ -257,6 +269,7 @@
257269
IPHONEOS_DEPLOYMENT_TARGET = 10.0;
258270
MTL_ENABLE_DEBUG_INFO = NO;
259271
SDKROOT = iphoneos;
272+
SWIFT_OBJC_INTERFACE_HEADER_NAME = "TNSMLKitCamera-Swift.h";
260273
TARGETED_DEVICE_FAMILY = "1,2";
261274
VALIDATE_PRODUCT = YES;
262275
VERSIONING_SYSTEM = "apple-generic";
@@ -267,6 +280,8 @@
267280
A3905B0E1EBCD37D008D0250 /* Debug */ = {
268281
isa = XCBuildConfiguration;
269282
buildSettings = {
283+
ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO;
284+
CLANG_ENABLE_MODULES = YES;
270285
CODE_SIGN_IDENTITY = "";
271286
DEFINES_MODULE = YES;
272287
DEVELOPMENT_TEAM = 8Q5F6M3TNS;
@@ -279,12 +294,16 @@
279294
PRODUCT_BUNDLE_IDENTIFIER = org.nativescript.plugin.firebase.MLKit;
280295
PRODUCT_NAME = "$(TARGET_NAME)";
281296
SKIP_INSTALL = YES;
297+
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
298+
SWIFT_VERSION = 3.0;
282299
};
283300
name = Debug;
284301
};
285302
A3905B0F1EBCD37D008D0250 /* Release */ = {
286303
isa = XCBuildConfiguration;
287304
buildSettings = {
305+
ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO;
306+
CLANG_ENABLE_MODULES = YES;
288307
CODE_SIGN_IDENTITY = "";
289308
DEFINES_MODULE = YES;
290309
DEVELOPMENT_TEAM = 8Q5F6M3TNS;
@@ -297,6 +316,7 @@
297316
PRODUCT_BUNDLE_IDENTIFIER = org.nativescript.plugin.firebase.MLKit;
298317
PRODUCT_NAME = "$(TARGET_NAME)";
299318
SKIP_INSTALL = YES;
319+
SWIFT_VERSION = 3.0;
300320
};
301321
name = Release;
302322
};

‎src/platforms/ios_lib/TNSMLKitCamera/TNSMLKitCamera/TNSMLKitCameraView.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@
1010
@property (nonatomic) UIImageOrientation imageOrientation;
1111

1212
- (nonnull id)initWithCaptureSession:(nonnull AVCaptureSession *)captureSession;
13+
+ (NSArray * _Nullable)resizeImage:(nonnull UIImage *)image;
1314

1415
@end

0 commit comments

Comments
 (0)