Make Facemesh work on iOS

Change-Id: Ia032137c1788c4672d646bc9c93c44bf0bf300bc
mit-cml · Sep 6, 2024 · 5f7650b · 5f7650b
1 parent 913eb7f
commit 5f7650b
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 30 deletions.
diff --git a/appinventor/AICompanionApp.xcodeproj/xcshareddata/xcschemes/AICompanionApp.xcscheme b/appinventor/AICompanionApp.xcodeproj/xcshareddata/xcschemes/AICompanionApp.xcscheme
@@ -86,23 +86,6 @@
             isEnabled = "YES">
          </EnvironmentVariable>
       </EnvironmentVariables>
-      <AdditionalOptions>
-         <AdditionalOption
-            key = "MallocStackLogging"
-            value = ""
-            isEnabled = "YES">
-         </AdditionalOption>
-         <AdditionalOption
-            key = "PrefersMallocStackLoggingLite"
-            value = ""
-            isEnabled = "YES">
-         </AdditionalOption>
-         <AdditionalOption
-            key = "MallocScribble"
-            value = ""
-            isEnabled = "YES">
-         </AdditionalOption>
-      </AdditionalOptions>
    </LaunchAction>
    <ProfileAction
       buildConfiguration = "Release"

diff --git a/appinventor/AIComponentKit.xcodeproj/project.pbxproj b/appinventor/AIComponentKit.xcodeproj/project.pbxproj
@@ -91,6 +91,8 @@
 		068E0EB71DC251BA0086FFDA /* Slider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 068E0EB61DC251BA0086FFDA /* Slider.swift */; };
 		068E0EB91DC251E90086FFDA /* Texting.swift in Sources */ = {isa = PBXBuildFile; fileRef = 068E0EB81DC251E90086FFDA /* Texting.swift */; };
 		06925559228B01B60091057E /* NSLock+Synchronized.swift in Sources */ = {isa = PBXBuildFile; fileRef = 06925558228B01B60091057E /* NSLock+Synchronized.swift */; };
+		06925B402C8B40D60085DC83 /* facemesh.min.js in Resources */ = {isa = PBXBuildFile; fileRef = 06925B3E2C8B40D60085DC83 /* facemesh.min.js */; };
+		06925B412C8B40D60085DC83 /* model.json in Resources */ = {isa = PBXBuildFile; fileRef = 06925B3F2C8B40D60085DC83 /* model.json */; };
 		06941F402603B4A900F8E1B2 /* StringUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = 06941F3F2603B4A900F8E1B2 /* StringUtil.swift */; };
 		06A47CD323F12EF400668126 /* PermissionException.swift in Sources */ = {isa = PBXBuildFile; fileRef = 06A47CD223F12EF400668126 /* PermissionException.swift */; };
 		06A623842507BC3200961743 /* Array+AIHelpers.swift in Sources */ = {isa = PBXBuildFile; fileRef = 06A623832507BC3200961743 /* Array+AIHelpers.swift */; };
@@ -355,6 +357,8 @@
 		068E0EB61DC251BA0086FFDA /* Slider.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Slider.swift; sourceTree = "<group>"; };
 		068E0EB81DC251E90086FFDA /* Texting.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Texting.swift; sourceTree = "<group>"; };
 		06925558228B01B60091057E /* NSLock+Synchronized.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "NSLock+Synchronized.swift"; sourceTree = "<group>"; };
+		06925B3E2C8B40D60085DC83 /* facemesh.min.js */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.javascript; path = facemesh.min.js; sourceTree = "<group>"; };
+		06925B3F2C8B40D60085DC83 /* model.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = model.json; sourceTree = "<group>"; };
 		06941F3F2603B4A900F8E1B2 /* StringUtil.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StringUtil.swift; sourceTree = "<group>"; };
 		06A47CD223F12EF400668126 /* PermissionException.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PermissionException.swift; sourceTree = "<group>"; };
 		06A623832507BC3200961743 /* Array+AIHelpers.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Array+AIHelpers.swift"; sourceTree = "<group>"; };
@@ -988,6 +992,8 @@
 		5F7266182C754755009C3CAB /* assets */ = {
 			isa = PBXGroup;
 			children = (
+				06925B3E2C8B40D60085DC83 /* facemesh.min.js */,
+				06925B3F2C8B40D60085DC83 /* model.json */,
 				5F7ED1012C35C94D004418C5 /* app.js */,
 				5F7ED1032C35C94F004418C5 /* group1-shard1of1.bin */,
 				5F7ED1022C35C94E004418C5 /* index.html */,
@@ -1194,6 +1200,7 @@
 			isa = PBXResourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				06925B402C8B40D60085DC83 /* facemesh.min.js in Resources */,
 				5F7ED1082C35C950004418C5 /* index.html in Resources */,
 				5F7ED1092C35C950004418C5 /* group1-shard1of1.bin in Resources */,
 				DE6D5A0F20FEA8DA003B2C67 /* checked.svg in Resources */,
@@ -1216,6 +1223,7 @@
 				DED066B21FE0846F009C8165 /* webview-error.html in Resources */,
 				061DC7B3223C396000539213 /* classic_button.pressed.png in Resources */,
 				DE6D5A0C20FE9E8D003B2C67 /* unchecked.svg in Resources */,
+				06925B412C8B40D60085DC83 /* model.json in Resources */,
 				062B743B2BC06AD9002E221F /* PrivacyInfo.xcprivacy in Resources */,
 				5F7ED10A2C35C950004418C5 /* VERSIONS.txt in Resources */,
 			);

diff --git a/appinventor/components-ios/src/Canvas.swift b/appinventor/components-ios/src/Canvas.swift
@@ -273,13 +273,20 @@ public class Canvas: ViewComponent, AbstractMethodsForViewComponent, UIGestureRe
     }
   }
 
-  @objc open func BackgroundImageinBase64(imageUrl: String) {
+  @objc open func setBackgroundImageinBase64(_ imageUrl: String) {
     if imageUrl.isEmpty {
       _imageSize = nil
       _backgroundImage = ""
       _backgroundImageView.image = nil
     } else {
-      let dataDecoded: NSData = NSData(base64Encoded: imageUrl, options: NSData.Base64DecodingOptions(rawValue: 0))!
+      let imageStr: String
+      if imageUrl.starts(with: "data:") {
+        let parts = imageUrl.split(";base64,")
+        imageStr = parts.last ?? ""
+      } else {
+        imageStr = imageUrl
+      }
+      let dataDecoded: NSData = NSData(base64Encoded: imageStr, options: NSData.Base64DecodingOptions(rawValue: 0))!
       guard let decodedImage: UIImage = UIImage(data: dataDecoded as Data) else {
         _imageSize = nil
         _backgroundImage = ""

diff --git a/appinventor/components-ios/src/assets/index.html b/appinventor/components-ios/src/assets/index.html
@@ -6,7 +6,23 @@
     <title>Tensorflow.js Extension</title>
     <script type="text/javascript">
         if (!window.FaceExtension) {
-          window.FaceExtension = { ready: function() {}, error: function(code, msg) {}, reportResult(result) {} };
+          const runtime = window.webkit.messageHandlers.FaceExtension;
+          let dispatch = (name) => {
+            return (...args) => {
+              const message = {
+                'functionCall': name,
+                'args': args
+              };
+              console.log(message);
+              runtime.postMessage(message);
+            }
+          }
+          window.FaceExtension = {
+            ready: dispatch('ready'),
+            error: dispatch('error'),
+            reportResult: dispatch('reportResult'),
+            reportImage: dispatch('reportImage')
+          };
         }
     </script>
 </head>

diff --git a/appinventor/components-ios/src/facemesh.swift b/appinventor/components-ios/src/facemesh.swift
@@ -124,7 +124,6 @@ fileprivate let MODEL_PATH_SUFFIX = ".tflite"
       }else {
         print("request not loaded")
       }
-      showImagePicker()
     }
   }
 
@@ -206,11 +205,10 @@ fileprivate let MODEL_PATH_SUFFIX = ".tflite"
     if message.name == "FaceExtension" {
       guard let dict = message.body as? [String: Any],
             let functionCall = dict["functionCall"] as? String,
-            let args = dict["args"] else {
+            let args = dict["args"] as? [Any] else {
         print("JSON Error message not received")
         return
       }
-      print(message.body)
       switch functionCall {
       case "ready":
         print("Model Ready")
@@ -222,20 +220,23 @@ fileprivate let MODEL_PATH_SUFFIX = ".tflite"
       case "reportResult":
         print("Reporting Result")
         do {
-          let result = try getYailObjectFromJson(args as? String, true)
-          print(result)
+          reportResult(result: args[0] as! String)
         } catch {
           print("Error parsing JSON from web view")
           Error(_ERROR_JSON_PARSE_FAILED as AnyObject, "Error parsing JSON from web view" as AnyObject)
         }
 
       case "reportImage":
         print("Reporting Image")
+        guard let dataUrl = args.first as? String else {
+          return
+        }
+        BackgroundImage = dataUrl
         VideoUpdated()
 
       case "error":
         print("Error function called")
-        if let errorDetails = args as? [String: Any],
+        if let errorDetails = args[0] as? [String: Any],
            let errorCode = errorDetails["errorCode"],
            let errorMessage = errorDetails["errorMessage"] {
           Error(errorCode as AnyObject, errorMessage as AnyObject)
@@ -395,19 +396,19 @@ fileprivate let MODEL_PATH_SUFFIX = ".tflite"
     return _keyPoints["rightEyeBottom"] ?? []
   }
 
-  @objc open var RightEarStart: [Double] {
+  @objc open var RightForehead: [Double] {
     return _keyPoints["rightEarStart"] ?? []
   }
 
-  @objc open var LeftEarStart: [Double] {
+  @objc open var LeftForehead: [Double] {
     return _keyPoints["leftEarStart"] ?? []
   }
 
-  @objc open var RightNoseTop: [Double] {
+  @objc open var RightNose: [Double] {
     return _keyPoints["rightNoseTop"] ?? []
   }
 
-  @objc open var LeftNoseTop: [Double] {
+  @objc open var LeftNose: [Double] {
     return _keyPoints["leftNoseTop"] ?? []
   }