Skip to content
This repository was archived by the owner on Apr 4, 2023. It is now read-only.

Commit e3aaeff

Browse files
EddyVerbruggenEddyVerbruggen
committed
#699 Add ML Kit support (refactored landmark recognition so it's more clear it's always Cloud based)
1 parent 5869baf commit e3aaeff

7 files changed

Lines changed: 32 additions & 132 deletions

File tree

‎demo-ng/app/tabs/mlkit/mlkit.component.html‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@
1010

1111
<Image row="2" [src]="pickedImage" width="240" horizontalAlignment="center" (tap)="reusePickedImage()"></Image>
1212

13-
<Label row="3" text="⬆️ Tap the image to use it again" textWrap="true" horizontalAlignment="center" *ngIf="pickedImage"></Label>
13+
<Label row="3" [text]="pickedImage ? '⬆️ Tap the image to use it again' : ''" textWrap="true" horizontalAlignment="center" class="m-b-20" width="100%"></Label>
14+
1415
</GridLayout>

‎demo-ng/app/tabs/mlkit/mlkit.component.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { ImageSource } from "tns-core-modules/image-source";
33

44
import { BarcodeFormat, MLKitScanBarcodesResult } from "nativescript-plugin-firebase/mlkit/barcodescanning";
55
import { MLKitRecognizeTextLocalResult, MLKitRecognizeTextCloudResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
6-
import { MLKitLandmarkRecognitionResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
6+
import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
77
import { MLKitDetectFacesResult } from "nativescript-plugin-firebase/mlkit/facedetection";
88
import { action } from "tns-core-modules/ui/dialogs";
99
import { ImageAsset } from "tns-core-modules/image-asset";
@@ -158,7 +158,7 @@ export class MLKitComponent {
158158
} else if (pickedItem === "Image labeling") {
159159
this.labelImage(imageSource);
160160
} else if (pickedItem === "Landmark recognition (cloud)") {
161-
this.recognizeLandmark(imageSource);
161+
this.recognizeLandmarkCloud(imageSource);
162162
}
163163
});
164164
}
@@ -193,11 +193,11 @@ export class MLKitComponent {
193193
.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
194194
}
195195

196-
private recognizeLandmark(imageSource: ImageSource): void {
197-
firebase.mlkit.landmarkrecognition.recognizeLandmark({
196+
private recognizeLandmarkCloud(imageSource: ImageSource): void {
197+
firebase.mlkit.landmarkrecognition.recognizeLandmarksCloud({
198198
image: imageSource
199199
}).then(
200-
(result: MLKitLandmarkRecognitionResult) => {
200+
(result: MLKitLandmarkRecognitionCloudResult) => {
201201
alert({
202202
title: `Result`,
203203
message: JSON.stringify(result.landmarks),

‎src/mlkit/index.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@ export interface MLKitOptions {
1111
image: Image | ImageSource;
1212
}
1313

14+
export type MLKitCloudModelType = "stable" | "latest";
15+
1416
export interface MLKitCloudOptions extends MLKitOptions {
1517
/**
1618
* Defaults to "stable".
1719
*/
18-
modelType?: "stable" | "latest";
20+
modelType?: MLKitCloudModelType;
1921
/**
2022
* Defaults to 10.
2123
*/

‎src/mlkit/landmarkrecognition/index.android.ts‎

Lines changed: 9 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,28 @@
11
import { ImageSource } from "tns-core-modules/image-source";
22
import { MLKitOptions, } from "../";
3-
import { MLKitLandmarkRecognitionOptions, MLKitLandmarkRecognitionResult } from "./";
4-
import { MLKitLandmarkRecognition as MLKitLandmarkRecognitionBase } from "./landmarkrecognition-common";
3+
import { MLKitLandmarkRecognitionCloudOptions, MLKitLandmarkRecognitionCloudResult } from "./index";
4+
import { MLKitCloudModelType } from "../index";
55

66
declare const com: any;
77

8-
export class MLKitLandmarkRecognition extends MLKitLandmarkRecognitionBase {
9-
10-
protected createDetector(): any {
11-
return getDetector(this.maxResults);
12-
}
13-
14-
protected createSuccessListener(): any {
15-
return new com.google.android.gms.tasks.OnSuccessListener({
16-
onSuccess: landmarks => {
17-
18-
if (landmarks.size() === 0) return;
19-
20-
// const imageSource = new ImageSource();
21-
// imageSource.setNativeSource(this.lastVisionImage.getBitmapForDebugging());
22-
23-
const result = <MLKitLandmarkRecognitionResult>{
24-
// imageSource: imageSource,
25-
landmarks: []
26-
};
27-
28-
for (let i = 0; i < landmarks.size(); i++) {
29-
const landmark = landmarks.get(i);
30-
result.landmarks.push({
31-
name: landmark.getLandmark(),
32-
confidence: landmark.getConfidence()
33-
});
34-
}
35-
36-
this.notify({
37-
eventName: MLKitLandmarkRecognition.scanResultEvent,
38-
object: this,
39-
value: result
40-
});
41-
}
42-
});
43-
}
44-
}
45-
46-
function getDetector(confidenceThreshold: number): any {
8+
function getDetector(modelType: MLKitCloudModelType, confidenceThreshold: number): any {
479
const landmarkDetectorOptions =
4810
new com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.Builder()
49-
.setMaxResults(confidenceThreshold)
11+
.setModelType(modelType === "latest" ? com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.LATEST_MODEL : com.google.firebase.ml.vision.cloud.FirebaseVisionCloudDetectorOptions.STABLE_MODEL)
12+
.setMaxResults(confidenceThreshold || 10)
5013
.build();
5114

5215
return com.google.firebase.ml.vision.FirebaseVision.getInstance().getVisionCloudLandmarkDetector(landmarkDetectorOptions);
5316
}
5417

55-
export function recognizeLandmark(options: MLKitLandmarkRecognitionOptions): Promise<MLKitLandmarkRecognitionResult> {
18+
export function recognizeLandmarksCloud(options: MLKitLandmarkRecognitionCloudOptions): Promise<MLKitLandmarkRecognitionCloudResult> {
5619
return new Promise((resolve, reject) => {
5720
try {
58-
const firebaseVisionLandmarkDetector = getDetector(options.maxResults || 10);
21+
const firebaseVisionLandmarkDetector = getDetector(options.modelType, options.maxResults);
5922

6023
const onSuccessListener = new com.google.android.gms.tasks.OnSuccessListener({
6124
onSuccess: landmarks => {
62-
const result = <MLKitLandmarkRecognitionResult>{
25+
const result = <MLKitLandmarkRecognitionCloudResult>{
6326
landmarks: []
6427
};
6528

@@ -86,7 +49,7 @@ export function recognizeLandmark(options: MLKitLandmarkRecognitionOptions): Pro
8649
.addOnFailureListener(onFailureListener);
8750

8851
} catch (ex) {
89-
console.log("Error in firebase.mlkit.recognizeLandmark: " + ex);
52+
console.log("Error in firebase.mlkit.recognizeLandmarksCloud: " + ex);
9053
reject(ex);
9154
}
9255
});

‎src/mlkit/landmarkrecognition/index.d.ts‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,14 @@
1-
import { MLKitCameraView, MLKitOptions, MLKitResult } from "../index";
1+
import { MLKitCloudOptions, MLKitResult } from "../index";
22

3-
export interface MLKitLandmarkRecognitionResult extends MLKitResult {
3+
export interface MLKitLandmarkRecognitionCloudResult extends MLKitResult {
44
// TODO add locations
55
landmarks: Array<{
66
name: string;
77
confidence: number;
88
}>;
99
}
1010

11-
export interface MLKitLandmarkRecognitionOptions extends MLKitOptions {
12-
/**
13-
* 10 by default
14-
*/
15-
maxResults?: number;
11+
export interface MLKitLandmarkRecognitionCloudOptions extends MLKitCloudOptions {
1612
}
1713

18-
export declare function recognizeLandmarks(options: MLKitLandmarkRecognitionOptions): Promise<MLKitLandmarkRecognitionResult>;
19-
20-
// TODO not sure this works.. but we need quite a processing threshold as this is always in the cloud..
21-
// .. and what about other features.. the camera should prolly always use a local model
22-
export declare class MLKitLandmarkRecognition extends MLKitCameraView {}
14+
export declare function recognizeLandmarksCloud(options: MLKitLandmarkRecognitionCloudOptions): Promise<MLKitLandmarkRecognitionCloudResult>;

‎src/mlkit/landmarkrecognition/index.ios.ts‎

Lines changed: 9 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,27 @@
11
import { ImageSource } from "tns-core-modules/image-source";
22
import { MLKitOptions } from "../";
3-
import { MLKitLandmarkRecognitionOptions, MLKitLandmarkRecognitionResult } from "./";
4-
import { MLKitLandmarkRecognition as MLKitLandmarkRecognitionBase } from "./landmarkrecognition-common";
3+
import { MLKitLandmarkRecognitionCloudOptions, MLKitLandmarkRecognitionCloudResult } from "./";
4+
import { MLKitCloudModelType } from "../index";
55

6-
export class MLKitLandmarkRecognition extends MLKitLandmarkRecognitionBase {
7-
8-
protected createDetector(): any {
9-
return getDetector(this.maxResults);
10-
}
11-
12-
protected createSuccessListener(): any {
13-
return (landmarks: NSArray<FIRVisionCloudLandmark>, error: NSError) => {
14-
if (error !== null) {
15-
console.log(error.localizedDescription);
16-
17-
} else if (landmarks !== null && landmarks.count > 0) {
18-
const result = <MLKitLandmarkRecognitionResult>{
19-
landmarks: []
20-
};
21-
22-
for (let i = 0, l = landmarks.count; i < l; i++) {
23-
const landmark: FIRVisionCloudLandmark = landmarks.objectAtIndex(i);
24-
console.log(">> detected landmark: " + landmark);
25-
result.landmarks.push({
26-
name: landmark.landmark,
27-
confidence: landmark.confidence
28-
});
29-
}
30-
31-
console.log(">>> notify " + MLKitLandmarkRecognition.scanResultEvent + " with " + JSON.stringify(result));
32-
this.notify({
33-
eventName: MLKitLandmarkRecognition.scanResultEvent,
34-
object: this,
35-
value: result
36-
});
37-
}
38-
}
39-
}
40-
41-
protected rotateRecording(): boolean {
42-
return false;
43-
}
44-
}
45-
46-
function getDetector(maxResults: number): FIRVisionCloudLandmarkDetector {
6+
function getDetector(modelType: MLKitCloudModelType, maxResults: number): FIRVisionCloudLandmarkDetector {
477
const firVision: FIRVision = FIRVision.vision();
488
const fIRVisionCloudDetectorOptions = FIRVisionCloudDetectorOptions.alloc();
49-
fIRVisionCloudDetectorOptions.maxResults = maxResults;
9+
fIRVisionCloudDetectorOptions.modelType = modelType === "latest" ? FIRVisionCloudModelType.Latest : FIRVisionCloudModelType.Stable;
10+
fIRVisionCloudDetectorOptions.maxResults = maxResults || 10;
5011
return firVision.cloudLandmarkDetectorWithOptions(fIRVisionCloudDetectorOptions);
5112
}
5213

53-
export function recognizeLandmark(options: MLKitLandmarkRecognitionOptions): Promise<MLKitLandmarkRecognitionResult> {
14+
export function recognizeLandmarksCloud(options: MLKitLandmarkRecognitionCloudOptions): Promise<MLKitLandmarkRecognitionCloudResult> {
5415
return new Promise((resolve, reject) => {
5516
try {
56-
const landmarkDetector = getDetector(options.maxResults || 10);
17+
const landmarkDetector = getDetector(options.modelType, options.maxResults);
5718

5819
landmarkDetector.detectInImageCompletion(getImage(options), (landmarks: NSArray<FIRVisionCloudLandmark>, error: NSError) => {
5920
if (error !== null) {
6021
reject(error.localizedDescription);
6122

6223
} else if (landmarks !== null) {
63-
const result = <MLKitLandmarkRecognitionResult>{
24+
const result = <MLKitLandmarkRecognitionCloudResult>{
6425
landmarks: []
6526
};
6627

@@ -77,7 +38,7 @@ export function recognizeLandmark(options: MLKitLandmarkRecognitionOptions): Pro
7738
}
7839
});
7940
} catch (ex) {
80-
console.log("Error in firebase.mlkit.recognizeLandmark: " + ex);
41+
console.log("Error in firebase.mlkit.recognizeLandmarksCloud: " + ex);
8142
reject(ex);
8243
}
8344
});

‎src/mlkit/landmarkrecognition/landmarkrecognition-common.ts‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

0 commit comments

Comments
 (0)