code changes completed for converting 8 demo apps to use lite version model

Author: jeffxtang

D2Go/D2Go.xcodeproj/project.pbxproj

@@ -8,13 +8,13 @@
 
 /* Begin PBXBuildFile section */
 		260F86432627C445008F2567 /* classes.txt in Resources */ = {isa = PBXBuildFile; fileRef = 260F86422627C445008F2567 /* classes.txt */; };
+		26A8C10A26DFEC4700F4A58D /* d2go_optimized.ptl in Resources */ = {isa = PBXBuildFile; fileRef = 26A8C10926DFEC4700F4A58D /* d2go_optimized.ptl */; };
 		26C3345C26210C3E00C37AD3 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26C3345B26210C3E00C37AD3 /* AppDelegate.swift */; };
 		26C3345E26210C3E00C37AD3 /* SceneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26C3345D26210C3E00C37AD3 /* SceneDelegate.swift */; };
 		26C3346026210C3E00C37AD3 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26C3345F26210C3E00C37AD3 /* ViewController.swift */; };
 		26C3346326210C3E00C37AD3 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 26C3346126210C3E00C37AD3 /* Main.storyboard */; };
 		26C3346526210C3F00C37AD3 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 26C3346426210C3F00C37AD3 /* Assets.xcassets */; };
 		26C3346826210C3F00C37AD3 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 26C3346626210C3F00C37AD3 /* LaunchScreen.storyboard */; };
-		26C3347626211FC900C37AD3 /* d2go_optimized.pt in Resources */ = {isa = PBXBuildFile; fileRef = 26C3347526211FC900C37AD3 /* d2go_optimized.pt */; };
 		26C3348C262120CB00C37AD3 /* PrePostProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26C3347D262120CB00C37AD3 /* PrePostProcessor.swift */; };
 		26C3348D262120CB00C37AD3 /* UIImage+Helper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26C3347E262120CB00C37AD3 /* UIImage+Helper.swift */; };
 		26C3348F262120CB00C37AD3 /* ObjectDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26C33483262120CB00C37AD3 /* ObjectDetector.swift */; };
@@ -30,6 +30,7 @@
 
 /* Begin PBXFileReference section */
 		260F86422627C445008F2567 /* classes.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = classes.txt; sourceTree = "<group>"; };
+		26A8C10926DFEC4700F4A58D /* d2go_optimized.ptl */ = {isa = PBXFileReference; lastKnownFileType = file; path = d2go_optimized.ptl; sourceTree = "<group>"; };
 		26C3345826210C3E00C37AD3 /* D2Go.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = D2Go.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		26C3345B26210C3E00C37AD3 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
 		26C3345D26210C3E00C37AD3 /* SceneDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SceneDelegate.swift; sourceTree = "<group>"; };
@@ -38,7 +39,6 @@
 		26C3346426210C3F00C37AD3 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
 		26C3346726210C3F00C37AD3 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
 		26C3346926210C3F00C37AD3 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
-		26C3347526211FC900C37AD3 /* d2go_optimized.pt */ = {isa = PBXFileReference; lastKnownFileType = file; path = d2go_optimized.pt; sourceTree = "<group>"; };
 		26C3347D262120CB00C37AD3 /* PrePostProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PrePostProcessor.swift; sourceTree = "<group>"; };
 		26C3347E262120CB00C37AD3 /* UIImage+Helper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "UIImage+Helper.swift"; sourceTree = "<group>"; };
 		26C33483262120CB00C37AD3 /* ObjectDetector.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ObjectDetector.swift; sourceTree = "<group>"; };
@@ -105,7 +105,7 @@
 				26C3348A262120CB00C37AD3 /* test1.png */,
 				26C33484262120CB00C37AD3 /* test2.jpg */,
 				26C3348B262120CB00C37AD3 /* test3.png */,
-				26C3347526211FC900C37AD3 /* d2go_optimized.pt */,
+				26A8C10926DFEC4700F4A58D /* d2go_optimized.ptl */,
 				260F86422627C445008F2567 /* classes.txt */,
 			);
 			path = D2Go;
@@ -204,10 +204,10 @@
 				26C33490262120CB00C37AD3 /* test2.jpg in Resources */,
 				26C3346526210C3F00C37AD3 /* Assets.xcassets in Resources */,
 				26C3346326210C3E00C37AD3 /* Main.storyboard in Resources */,
-				26C3347626211FC900C37AD3 /* d2go_optimized.pt in Resources */,
 				26C33495262120CB00C37AD3 /* test1.png in Resources */,
 				26C33496262120CB00C37AD3 /* test3.png in Resources */,
 				260F86432627C445008F2567 /* classes.txt in Resources */,
+				26A8C10A26DFEC4700F4A58D /* d2go_optimized.ptl in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -401,9 +401,9 @@
 					"-l\"torch_cpu\"",
 					"-l\"torchvision_ops\"",
 					"-force_load",
-					"$(PODS_ROOT)/LibTorch/install/lib/libtorch.a",
+					"$(PODS_ROOT)/LibTorch-Lite/install/lib/libtorch.a",
 					"-force_load",
-					"$(PODS_ROOT)/LibTorch/install/lib/libtorch_cpu.a",
+					"$(PODS_ROOT)/LibTorch-Lite/install/lib/libtorch_cpu.a",
 					"-force_load",
 					"$(PODS_ROOT)/LibTorchvision/install/lib/libtorchvision_ops.a",
 				);
@@ -445,9 +445,9 @@
 					"-l\"torch_cpu\"",
 					"-l\"torchvision_ops\"",
 					"-force_load",
-					"$(PODS_ROOT)/LibTorch/install/lib/libtorch.a",
+					"$(PODS_ROOT)/LibTorch-Lite/install/lib/libtorch.a",
 					"-force_load",
-					"$(PODS_ROOT)/LibTorch/install/lib/libtorch_cpu.a",
+					"$(PODS_ROOT)/LibTorch-Lite/install/lib/libtorch_cpu.a",
 					"-force_load",
 					"$(PODS_ROOT)/LibTorchvision/install/lib/libtorchvision_ops.a",
 				);

D2Go/D2Go/Inference/InferenceModule.h

@@ -5,6 +5,7 @@
 // LICENSE file in the root directory of this source tree.
 
 #import <Foundation/Foundation.h>
+#import <UIKit/UIKit.h>
 
 NS_ASSUME_NONNULL_BEGIN
 

D2Go/D2Go/Inference/InferenceModule.mm

@@ -5,23 +5,22 @@
 // LICENSE file in the root directory of this source tree.
 
 #import "InferenceModule.h"
-#import <LibTorch.h>
+#import <Libtorch-Lite/Libtorch-Lite.h>
 
 const int input_width = 640;
 const int input_height = 640;
 const int threshold = 0.5;
 
 
 @implementation InferenceModule {
-    @protected torch::jit::script::Module _impl;
+    @protected torch::jit::mobile::Module _impl;
 }
 
 - (nullable instancetype)initWithFileAtPath:(NSString*)filePath {
     self = [super init];
     if (self) {
         try {
-            _impl = torch::jit::load(filePath.UTF8String);
-            _impl.eval();
+            _impl = torch::jit::_load_for_mobile(filePath.UTF8String);
         } catch (const std::exception& exception) {
             NSLog(@"%s", exception.what());
             return nil;
@@ -39,7 +38,12 @@ - (nullable instancetype)initWithFileAtPath:(NSString*)filePath {
         std::vector<torch::Tensor> v;
         v.push_back(tensor);
 
+        
+        CFTimeInterval startTime = CACurrentMediaTime();
         auto outputTuple = _impl.forward({ at::TensorList(v) }).toTuple();
+        CFTimeInterval elapsedTime = CACurrentMediaTime() - startTime;
+        NSLog(@"inference time:%f", elapsedTime);
+
 
         auto outputDict = outputTuple->elements()[1].toList().get(0).toGenericDict();
         auto boxesTensor = outputDict.at("boxes").toTensor();

D2Go/D2Go/Inference/ObjectDetector.swift

@@ -8,7 +8,7 @@ import UIKit
 
 class ObjectDetector {
     lazy var module: InferenceModule = {
-        if let filePath = Bundle.main.path(forResource: "d2go_optimized", ofType: "pt"),
+        if let filePath = Bundle.main.path(forResource: "d2go_optimized", ofType: "ptl"),
             let module = InferenceModule(fileAtPath: filePath) {
             return module
         } else {

D2Go/D2Go/d2go_optimized.pt

@@ -4,7 +4,7 @@ target 'D2Go' do
   # Comment the next line if you don't want to use dynamic frameworks
   use_frameworks!
 
-  pod 'LibTorch', '1.9.0'
+  pod 'LibTorch-Lite', '~>1.9.0'
   pod 'LibTorchvision', '0.10.0'
 
 end

D2Go/D2Go/d2go_optimized.ptl

@@ -70,6 +70,7 @@ def forward(self, inputs: List[torch.Tensor]):
     scripted_model = torch.jit.script(wrapped_model)
     optimized_model = optimize_for_mobile(scripted_model)
     optimized_model.save("D2Go/d2go_optimized.pt")
+    optimized_model._save_for_lite_interpreter("D2Go/d2go_optimized.ptl")
 
 if __name__ == '__main__':
     test_export_torchvision_format()

D2Go/Podfile

@@ -20,7 +20,6 @@ - (nullable instancetype)initWithFileAtPath:(NSString*)filePath {
     if (self) {
         try {
             _impl = torch::jit::_load_for_mobile(filePath.UTF8String);
-            _impl.eval();
         } catch (const std::exception& exception) {
             NSLog(@"%s", exception.what());
             return nil;
@@ -40,9 +39,6 @@ - (unsigned char*)segmentImage:(void *)imageBuffer withWidth:(int)width withHeig
 
         at::Tensor tensor = torch::from_blob(imageBuffer, { 1, 3, width, height }, at::kFloat);
 
-        torch::autograd::AutoGradMode guard(false);
-        at::AutoNonVariableTypeMode non_var_type_mode(true);
-
         float* floatInput = tensor.data_ptr<float>();
         if (!floatInput) {
             return nil;
@@ -52,8 +48,13 @@ - (unsigned char*)segmentImage:(void *)imageBuffer withWidth:(int)width withHeig
             [inputs addObject:@(floatInput[i])];
         }
 
+        c10::InferenceMode guard;
+        
+        CFTimeInterval startTime = CACurrentMediaTime();
         auto outputDict = _impl.forward({ tensor }).toGenericDict();
-
+        CFTimeInterval elapsedTime = CACurrentMediaTime() - startTime;
+        NSLog(@"inference time:%f", elapsedTime);
+        
         auto outputTensor = outputDict.at("out").toTensor();
 
         float* floatBuffer = outputTensor.data_ptr<float>();

D2Go/create_d2go.py

@@ -1,7 +1,10 @@
 import torch
+from torch.utils.mobile_optimizer import optimize_for_mobile
 
 model = torch.hub.load('pytorch/vision:v0.9.0', 'deeplabv3_resnet50', pretrained=True)
 model.eval()
 
 scripted_module = torch.jit.script(model)
-scripted_module._save_for_lite_interpreter("ImageSegmentation/deeplabv3_scripted.ptl")
+optimized_model = optimize_for_mobile(scripted_module)
+optimized_model.save("ImageSegmentation/deeplabv3_scripted.pt")
+optimized_model._save_for_lite_interpreter("ImageSegmentation/deeplabv3_scripted.ptl")