Merge pull request #54 from jeffxtang/lite_beta

Updated for Image Segmentation Lite Interpreter demo app

Author: xta0

ImageSegmentation/ImageSegmentation.xcodeproj/project.pbxproj

@@ -14,11 +14,11 @@
 		265BAFEF253A6A6800467AC4 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 265BAFED253A6A6800467AC4 /* Main.storyboard */; };
 		265BAFF1253A6A6900467AC4 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 265BAFF0253A6A6900467AC4 /* Assets.xcassets */; };
 		265BAFF4253A6A6900467AC4 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 265BAFF2253A6A6900467AC4 /* LaunchScreen.storyboard */; };
-		265BB005253A6B6200467AC4 /* deeplabv3_scripted.pt in Resources */ = {isa = PBXBuildFile; fileRef = 265BB004253A6B6200467AC4 /* deeplabv3_scripted.pt */; };
 		265BB008253A6B9600467AC4 /* deeplab.jpg in Resources */ = {isa = PBXBuildFile; fileRef = 265BB007253A6B9600467AC4 /* deeplab.jpg */; };
 		265BB00E253A6E0E00467AC4 /* UIImage+Helper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 265BB00D253A6E0E00467AC4 /* UIImage+Helper.swift */; };
 		265BB017253A7F0500467AC4 /* TorchModule.mm in Sources */ = {isa = PBXBuildFile; fileRef = 265BB015253A7F0500467AC4 /* TorchModule.mm */; };
 		265F9A6F2551CB3700B8F2EC /* dog.jpg in Resources */ = {isa = PBXBuildFile; fileRef = 265F9A6E2551CB3700B8F2EC /* dog.jpg */; };
+		266A451D267974C300548578 /* deeplabv3_scripted.ptl in Resources */ = {isa = PBXBuildFile; fileRef = 266A451C267974C300548578 /* deeplabv3_scripted.ptl */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXFileReference section */
@@ -33,12 +33,12 @@
 		265BAFF3253A6A6900467AC4 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
 		265BAFF5253A6A6900467AC4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
 		265BAFFF253A6B1200467AC4 /* ImageSegmentation-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "ImageSegmentation-Bridging-Header.h"; sourceTree = "<group>"; };
-		265BB004253A6B6200467AC4 /* deeplabv3_scripted.pt */ = {isa = PBXFileReference; lastKnownFileType = file; path = deeplabv3_scripted.pt; sourceTree = "<group>"; };
 		265BB007253A6B9600467AC4 /* deeplab.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = deeplab.jpg; sourceTree = "<group>"; };
 		265BB00D253A6E0E00467AC4 /* UIImage+Helper.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "UIImage+Helper.swift"; sourceTree = "<group>"; };
 		265BB015253A7F0500467AC4 /* TorchModule.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = TorchModule.mm; sourceTree = "<group>"; };
 		265BB016253A7F0500467AC4 /* TorchModule.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = TorchModule.h; sourceTree = "<group>"; };
 		265F9A6E2551CB3700B8F2EC /* dog.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = dog.jpg; sourceTree = "<group>"; };
+		266A451C267974C300548578 /* deeplabv3_scripted.ptl */ = {isa = PBXFileReference; lastKnownFileType = file; path = deeplabv3_scripted.ptl; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -92,7 +92,7 @@
 				265BB015253A7F0500467AC4 /* TorchModule.mm */,
 				265BB00D253A6E0E00467AC4 /* UIImage+Helper.swift */,
 				265BAFFF253A6B1200467AC4 /* ImageSegmentation-Bridging-Header.h */,
-				265BB004253A6B6200467AC4 /* deeplabv3_scripted.pt */,
+				266A451C267974C300548578 /* deeplabv3_scripted.ptl */,
 				265BB007253A6B9600467AC4 /* deeplab.jpg */,
 				265F9A6E2551CB3700B8F2EC /* dog.jpg */,
 			);
@@ -161,8 +161,8 @@
 				265BAFF4253A6A6900467AC4 /* LaunchScreen.storyboard in Resources */,
 				265BB008253A6B9600467AC4 /* deeplab.jpg in Resources */,
 				265BAFF1253A6A6900467AC4 /* Assets.xcassets in Resources */,
-				265BB005253A6B6200467AC4 /* deeplabv3_scripted.pt in Resources */,
 				265BAFEF253A6A6800467AC4 /* Main.storyboard in Resources */,
+				266A451D267974C300548578 /* deeplabv3_scripted.ptl in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

ImageSegmentation/ImageSegmentation/TorchModule.mm

@@ -8,18 +8,18 @@
 #import "UIImageHelper.h"
 #import <CoreImage/CoreImage.h>
 #import <ImageIO/ImageIO.h>
-#import <LibTorch/LibTorch.h>
+#import <Libtorch-Lite/Libtorch-Lite.h>
 
 @implementation TorchModule {
 @protected
-    torch::jit::script::Module _impl;
+    torch::jit::mobile::Module _impl;
 }
 
 - (nullable instancetype)initWithFileAtPath:(NSString*)filePath {
     self = [super init];
     if (self) {
         try {
-            _impl = torch::jit::load(filePath.UTF8String);
+            _impl = torch::jit::_load_for_mobile(filePath.UTF8String);
             _impl.eval();
         } catch (const std::exception& exception) {
             NSLog(@"%s", exception.what());

ImageSegmentation/ImageSegmentation/ViewController.swift

@@ -10,7 +10,7 @@ class ViewController: UIViewController {
 
     private lazy var module: TorchModule = {
         if let filePath = Bundle.main.path(forResource:
-            "deeplabv3_scripted", ofType: "pt"),
+            "deeplabv3_scripted", ofType: "ptl"),
             let module = TorchModule(fileAtPath: filePath) {
             return module
         } else {

ImageSegmentation/Podfile

@@ -6,5 +6,5 @@ target 'ImageSegmentation' do
   use_frameworks!
 
   # Pods for ImageSegmentation
-  pod 'LibTorch', '~>1.7.0'
+  pod 'LibTorch-Lite', '~>1.9.0'
 end

ImageSegmentation/README.md

@@ -1,8 +1,15 @@
-# Semantic Image Segmentation DeepLabV3 on iOS
+# Semantic Image Segmentation DeepLabV3 with Mobile Interpreter on iOS
 
 ## Introduction
 
-This repo offers a Python script that converts the [PyTorch DeepLabV3 model](https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101) for mobile apps and an iOS app that uses the model to segment images.
+This repo offers a Python script that converts the [PyTorch DeepLabV3 model](https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101) to the Mobile Interpreter version and an iOS app that uses the model to segment images.
+
+## Prerequisites
+
+* PyTorch 1.9.0 and torchvision 0.10.0 (Optional)
+* Python 3.8 or above (Optional)
+* iOS Cocoapods LibTorch-Lite 1.9.0
+* Xcode 12.4 or later
 
 ## Quick Start
 
@@ -14,19 +21,19 @@ If you don't have the PyTorch environment set up to run the script below to gene
 
 Be aware that the downloadable model file was created with PyTorch 1.7.0, matching the iOS LibTorch library 1.7.0 specified in the `Podfile`. If you use a different version of PyTorch to create your model by following the instructions below, make sure you specify the same iOS LibTorch version in the `Podfile` to avoid possible errors caused by the version mismatch. Furthermore, if you want to use the latest prototype features in the PyTorch master branch to create the model, follow the steps at [Building PyTorch iOS Libraries from Source](https://pytorch.org/mobile/ios/#build-pytorch-ios-libraries-from-source) on how to use the model in iOS.
 
-Open a Mac Terminal, run the following commands:
+Open a Mac Terminal, first install PyTorch 1.9.0 and torchvision 0.10.0 using command like `pip install torch torchvision`, then run the following commands:
 
 ```
 git clone https://github.com/pytorch/ios-demo-app
 cd ios-demo-app/ImageSegmentation
 python deeplabv3.py
 ```
 
-The Python script `deeplabv3.py` is used to generate the TorchScript-formatted model for mobile apps. Then run `mv deeplabv3_scripted.pt ImageSegmentation` to move the model file to the right location.
+The Python script `deeplabv3.py` is used to generate the Lite Interpreter model file `deeplabv3_scripted.ptl` to be used in iOS.
 
 ### 2. Use LibTorch
 
-Run the commands below:
+Run the commands below (note the `Podfile` uses `pod 'LibTorch-Lite', '~>1.9.0'`):
 
 ```
 pod install
@@ -36,8 +43,6 @@ open ImageSegmentation.xcworkspace/
 ### 3. Run the app
 Select an iOS simulator or device on Xcode to run the app. The example image and its segmented result are as follows:
 
-results are:
-
 ![](screenshot1.png)
 ![](screenshot2.png)
 
@@ -46,3 +51,5 @@ Note that the `resized` method in `UIImage+Helper.swift` is used to speed up the
 ## Tutorial
 
 Read the tutorial [here](https://pytorch.org/tutorials/beginner/deeplabv3_on_ios.html) for detailed step-by-step instructions of how to prepare and run the [PyTorch DeepLabV3 model](https://pytorch.org/hub/pytorch_vision_deeplabv3_resnet101) on iOS, as well as practical tips on how to successfully use a pre-trained PyTorch model on iOS and avoid common pitfalls.
+
+For more information on using Mobile Interpreter in Android, see the tutorial [here](https://pytorch.org/tutorials/recipes/mobile_interpreter.html).

ImageSegmentation/deeplabv3.py

@@ -1,8 +1,7 @@
 import torch
 
-model = torch.hub.load('pytorch/vision:v0.7.0', 'deeplabv3_resnet50', pretrained=True)
+model = torch.hub.load('pytorch/vision:v0.9.0', 'deeplabv3_resnet50', pretrained=True)
 model.eval()
 
-scriptedm = torch.jit.script(model)
-torch.jit.save(scriptedm, "deeplabv3_scripted.pt")
-
+scripted_module = torch.jit.script(model)
+scripted_module._save_for_lite_interpreter("ImageSegmentation/deeplabv3_scripted.ptl")

SpeechRecognition/README.md

@@ -31,12 +31,13 @@ Be aware that the downloadable model file was created with PyTorch 1.9.0 and tor
 
 ### 2. Prepare the Model
 
-To install PyTorch 1.9.0 and torchvision 0.10.0, you can do something like this:
+To install PyTorch 1.9.0, torchaudio 0.9.0 and the Hugging Face transformers, you can do something like this:
 
 ```
 conda create -n wav2vec2 python=3.8.5
 conda activate wav2vec2
-pip install torch torchvision
+pip install torch torchaudio
+pip install transformers
 ```
 
 Now with PyTorch 1.9.0 and torchaudio 0.9.0 installed, run the following commands on a Terminal:
@@ -58,7 +59,7 @@ open SpeechRecognition.xcworkspace/
 
 ### 3. Build and run with Xcode
 
-After the app runs, tap the Start button and start saying something; after 12 seconds (you can change `private let AUDIO_LEN_IN_SECOND = 12` in `ViewController.swift` for the recording length), the model will infer to recognize your speech. Some example results are as follows:
+After the app runs, tap the Start button and start saying something; after 12 seconds (you can change `private let AUDIO_LEN_IN_SECOND = 12` in `ViewController.swift` for a longer or shorter recording length), the model will infer to recognize your speech. Some example results are as follows:
 
 ![](screenshot1.png)
 ![](screenshot2.png)