Add support for ML Kit's Smart Reply feature #1219

Author: EddyVerbruggen

demo-ng/app/tabs/mlkit/mlkit.component.ts

@@ -11,6 +11,7 @@ import {
 } from "nativescript-plugin-firebase/mlkit/imagelabeling";
 import { MLKitLandmarkRecognitionCloudResult } from "nativescript-plugin-firebase/mlkit/landmarkrecognition";
 import { MLKitNaturalLanguageIdentificationResult } from "nativescript-plugin-firebase/mlkit/naturallanguageidentification";
+import { MLKitSmartReplyConversationMessage } from "nativescript-plugin-firebase/mlkit/smartreply";
 import { MLKitRecognizeTextResult } from "nativescript-plugin-firebase/mlkit/textrecognition";
 import * as fileSystemModule from "tns-core-modules/file-system";
 import { ImageAsset } from "tns-core-modules/image-asset";
@@ -38,7 +39,8 @@ export class MLKitComponent {
     "Image labeling (cloud)",
     "Custom model",
     "Landmark recognition (cloud)",
-    "Language identification (on device)"
+    "Language identification",
+    "Smart Reply"
   ];
 
   private mlkitOnDeviceFeatures: Array<string> = [
@@ -191,8 +193,10 @@ export class MLKitComponent {
         this.recognizeLandmarkCloud(imageSource);
       } else if (pickedItem === "Custom model") {
         this.customModel(imageSource);
-      } else if (pickedItem === "Language identification (on device)") {
+      } else if (pickedItem === "Language identification") {
         this.languageIdentification(imageSource);
+      } else if (pickedItem === "Smart reply") {
+        this.smartReply(imageSource);
       }
     });
   }
@@ -246,21 +250,41 @@ export class MLKitComponent {
     // First recognize text, then get its language
     firebase.mlkit.textrecognition.recognizeTextOnDevice({
       image: imageSource
-    }).then(
-        (result: MLKitRecognizeTextResult) => {
-          firebase.mlkit.naturallanguageidentification.identifyNaturalLanguage({
-            text: result.text
-          }).then(
-              (languageIdResult: MLKitNaturalLanguageIdentificationResult) => {
-                alert({
-                  title: `Result`,
-                  message: `Language code: ${languageIdResult.languageCode}`,
-                  okButtonText: "OK"
-                });
-              })
-              .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
-        })
-        .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+    }).then((result: MLKitRecognizeTextResult) => {
+      firebase.mlkit.naturallanguageidentification.identifyNaturalLanguage({
+        text: result.text
+      }).then((languageIdResult: MLKitNaturalLanguageIdentificationResult) => {
+        alert({
+          title: `Result`,
+          message: `Language code: ${languageIdResult.languageCode}`,
+          okButtonText: "OK"
+        });
+      }).catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+    }).catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+  }
+
+  // it would be easier to hardcode the conversation, but this fits better with the other image-based examples
+  private smartReply(imageSource: ImageSource): void {
+    firebase.mlkit.textrecognition.recognizeTextOnDevice({
+      image: imageSource
+    }).then((result: MLKitRecognizeTextResult) => {
+      const messages: Array<MLKitSmartReplyConversationMessage> = [];
+      result.blocks.forEach(block => messages.push({
+        text: block.text,
+        userId: "abc",
+        localUser: false,
+        timestamp: new Date().getTime()
+      }));
+      firebase.mlkit.smartreply.suggestReplies({
+        messages
+      }).then((result: Array<string>) => {
+        alert({
+          title: `Suggestions`,
+          message: JSON.stringify(result),
+          okButtonText: "OK"
+        });
+      }).catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+    }).catch(errorMessage => console.log("ML Kit error: " + errorMessage));
   }
 
   private customModel(imageSource: ImageSource): void {

docs/ML_KIT.md

@@ -84,6 +84,8 @@ To be able to use Cloud features you need to do two things:
 |[Barcode scanning](#barcode-scanning)|✅|
 |[Image labeling](#image-labeling)|✅|✅
 |[Landmark recognition](#landmark-recognition)||✅
+|[Natural language identification](#natural-language-identification)|✅|
+|[Smart reply](#smart-reply)|✅|
 |[Custom model inference](#custom-model-inference)|✅|✅
 
 ### Text recognition
@@ -384,6 +386,59 @@ firebase.mlkit.landmarkrecognition.recognizeLandmarksCloud({
 .catch(errorMessage => console.log("ML Kit error: " + errorMessage));
 ```
 
+### Natural language identification
+<img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_language_detection.png" height="153px" alt="ML Kit - Landmark detection"/>
+
+[Firebase documentation 🌎](https://firebase.google.com/docs/ml-kit/identify-languages)
+
+#### Still image
+Note that you could hook this up to text recognition running from a live camera stream, to get realtime results.
+Check [demo-ng](demo-ng) if you're interested in such a solution.
+
+```typescript
+import { MLKitNaturalLanguageIdentificationResult } from "nativescript-plugin-firebase/mlkit/naturallanguageidentification";
+const firebase = require("nativescript-plugin-firebase");
+
+firebase.mlkit.naturallanguageidentification.identifyNaturalLanguage({
+  text: "Some text to detect the language for"
+})
+.then((languageIdResult: MLKitNaturalLanguageIdentificationResult) => console.log(`Language code: ${languageIdResult.languageCode}`))
+.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+```
+
+### Smart reply
+<img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_smart_reply.png" height="153px" alt="ML Kit - Smart reply"/>
+
+[Firebase documentation 🌎](https://firebase.google.com/docs/ml-kit/generate-smart-replies)
+
+#### Still image
+
+```typescript
+import { MLKitSmartReplyConversationMessage } from "nativescript-plugin-firebase/mlkit/smartreply";
+const firebase = require("nativescript-plugin-firebase");
+
+// build a converstation history MLKit can create suggestions for, in chronological order
+const conversation: Array<MLKitSmartReplyConversationMessage> = [];
+conversation.push({
+  text: "some text",
+  userId: "abc",
+  localUser: false,
+  timestamp: new Date().getTime()
+});
+
+conversation.push({
+  text: "some other text",
+  userId: "def",
+  localUser: true,
+  timestamp: new Date().getTime()
+});
+
+firebase.mlkit.smartreply.suggestReplies({
+  conversation
+}).then((suggestions: Array<string>) => console.log(JSON.stringify(suggestions)))
+.catch(errorMessage => console.log("ML Kit error: " + errorMessage));
+```
+
 ### Custom model inference
 <img src="https://raw.githubusercontent.com/EddyVerbruggen/nativescript-plugin-firebase/master/docs/images/features/mlkit_custom_model_tflite.png" height="153px" alt="ML Kit - Custom Model (TensorFlow Lite)"/>
 

docs/images/features/mlkit_language_detection.png

@@ -5,6 +5,7 @@ import * as imagelabeling from "./imagelabeling";
 import * as landmarkrecognition from "./landmarkrecognition";
 import * as custommodel from "./custommodel";
 import * as naturallanguageidentification from "./naturallanguageidentification";
+import * as smartreply from "./smartreply";
 
 import { ImageSource } from "tns-core-modules/image-source";
 import { Image } from "tns-core-modules/ui/image";
@@ -44,6 +45,7 @@ export {
   imagelabeling,
   landmarkrecognition,
   custommodel,
-  naturallanguageidentification
+  naturallanguageidentification,
+  smartreply
 };
 

docs/images/features/mlkit_smart_reply.png

@@ -0,0 +1,46 @@
+import { MLKitSmartReplySuggestRepliesOptions, } from "./index";
+
+export function suggestReplies(options: MLKitSmartReplySuggestRepliesOptions): Promise<Array<string>> {
+  return new Promise((resolve, reject) => {
+    try {
+      const conversation: java.util.List<com.google.firebase.ml.naturallanguage.smartreply.FirebaseTextMessage> = new java.util.ArrayList();
+
+      options.conversation.forEach(m => {
+        if (m.localUser) {
+          conversation.add(com.google.firebase.ml.naturallanguage.smartreply.FirebaseTextMessage.createForLocalUser(m.text, m.timestamp))
+        } else {
+          conversation.add(com.google.firebase.ml.naturallanguage.smartreply.FirebaseTextMessage.createForRemoteUser(m.text, m.timestamp, m.userId))
+        }
+      });
+
+      const smartReply: com.google.firebase.ml.naturallanguage.smartreply.FirebaseSmartReply =
+          com.google.firebase.ml.naturallanguage.FirebaseNaturalLanguage.getInstance().getSmartReply();
+
+      smartReply.suggestReplies(conversation)
+          .addOnSuccessListener(new (<any>com.google.android.gms).tasks.OnSuccessListener({
+            onSuccess: (result: com.google.firebase.ml.naturallanguage.smartreply.SmartReplySuggestionResult) => {
+              if (result.getStatus() == com.google.firebase.ml.naturallanguage.smartreply.SmartReplySuggestionResult.STATUS_NOT_SUPPORTED_LANGUAGE) {
+                reject("Unsupported language");
+              } else if (result.getStatus() == com.google.firebase.ml.naturallanguage.smartreply.SmartReplySuggestionResult.STATUS_NO_REPLY) {
+                reject("No reply");
+              } else if (result.getStatus() == com.google.firebase.ml.naturallanguage.smartreply.SmartReplySuggestionResult.STATUS_SUCCESS) {
+                const suggestions = [];
+                for (let i = 0; i < result.getSuggestions().size(); i++) {
+                  const s = result.getSuggestions().get(i);
+                  suggestions.push(s.getText());
+                }
+                resolve(suggestions);
+              } else {
+                reject();
+              }
+            }
+          }))
+          .addOnFailureListener(new (<any>com.google.android.gms).tasks.OnFailureListener({
+            onFailure: exception => reject(exception.getMessage())
+          }));
+    } catch (ex) {
+      console.log("Error in firebase.mlkit.suggestReplies: " + ex);
+      reject(ex);
+    }
+  });
+}

src/mlkit/index.ts

@@ -0,0 +1,12 @@
+export interface MLKitSmartReplyConversationMessage {
+  text: string;
+  timestamp: number;
+  userId: string;
+  localUser: boolean;
+}
+
+export interface MLKitSmartReplySuggestRepliesOptions {
+  conversation: Array<MLKitSmartReplyConversationMessage>;
+}
+
+export declare function suggestReplies(options: MLKitSmartReplySuggestRepliesOptions): Promise<Array<string>>;

src/mlkit/smartreply/index.android.ts

@@ -0,0 +1,41 @@
+import { MLKitSmartReplySuggestRepliesOptions, } from "./index";
+
+export function suggestReplies(options: MLKitSmartReplySuggestRepliesOptions): Promise<Array<string>> {
+  return new Promise((resolve, reject) => {
+    try {
+      const naturalLanguage = FIRNaturalLanguage.naturalLanguage();
+      const smartReply = naturalLanguage.smartReply();
+      const conversation: Array<FIRTextMessage> = [];
+      options.conversation.forEach(m => conversation.push(new FIRTextMessage({
+        text: m.text,
+        isLocalUser: m.localUser,
+        timestamp: m.timestamp,
+        userID: m.userId
+      })));
+
+      smartReply.suggestRepliesForMessagesCompletion(conversation, (result: FIRSmartReplySuggestionResult, error: NSError) => {
+        if (error) {
+          reject(error.localizedDescription);
+        } else if (!result) {
+          reject("No results");
+        } else if (result.status === FIRSmartReplyResultStatus.NotSupportedLanguage) {
+          reject("Unsupported language");
+        } else if (result.status === FIRSmartReplyResultStatus.NoReply) {
+          reject("No reply");
+        } else if (result.status === FIRSmartReplyResultStatus.Success) {
+          const suggestions = [];
+          for (let i = 0; i < result.suggestions.count; i++) {
+            const s = result.suggestions.objectAtIndex(i);
+            suggestions.push(s.text);
+          }
+          resolve(suggestions);
+        } else {
+          reject();
+        }
+      });
+    } catch (ex) {
+      console.log("Error in firebase.mlkit.suggestReplies: " + ex);
+      reject(ex);
+    }
+  });
+}

src/mlkit/smartreply/index.d.ts

@@ -4184,6 +4184,9 @@ const supportsIOSModernBuildSystem = tnsVersionFull.indexOf("5.2.0-") > -1 || se
 // Custom gradle buildscripts are supported from NativeScript-Android version 5.3.0 (TODO this actually checks the CLI version)
 const supportsGradleBuildscripts = tnsVersionFull.indexOf("5.3.0-") > -1 || semver.gte(tnsVersionFull, "5.3.0");
 
+// TODO use this when writing stuff like 'com.android.support' below, because {N} 6 will have to use AndroidX
+const supportsAndroidX = semver.gte(tnsVersionFull, "6.0.0");
+
 if (!supportsIOSModernBuildSystem) {
   console.log(`You're using NativeScript ${tnsVersionFull}.. which doesn't support the latest Firestore and in-app-messaging SDKs. Upgrade NativeScript to at least 5.2.0 if you need those!\n\n`);
 }