diff --git a/README.md b/README.md index 1c0c0816e..06f3c50e2 100644 --- a/README.md +++ b/README.md @@ -13,19 +13,10 @@ API -

 

- -

- - Current React Native PyTorch Core npm package version. - -

- -

- - PlayTorch is released under the MIT license. - -

+[![npm version](http://img.shields.io/npm/v/react-native-playtorch.svg?style=flat-square)](https://npmjs.org/package/react-native-playtorch "View this project on npm") +[![npm downloads](http://img.shields.io/npm/dm/react-native-playtorch.svg?style=flat-square)](https://npmjs.org/package/react-native-playtorch "View this project on npm") +[![npm licence](http://img.shields.io/npm/l/react-native-playtorch.svg?style=flat-square)](https://npmjs.org/package/react-native-playtorch "View this project on npm") +[![Platform](https://img.shields.io/badge/platform-ios%20%7C%20android-989898.svg?style=flat-square)](https://npmjs.org/package/react-native-playtorch "View this project on npm") ---------------------- diff --git a/react-native-pytorch-core/README.md b/react-native-pytorch-core/README.md index bc7e3cac0..4ede7ead3 100644 --- a/react-native-pytorch-core/README.md +++ b/react-native-pytorch-core/README.md @@ -1,5 +1,10 @@ # PyTorch core library for React Native +[![npm version](http://img.shields.io/npm/v/react-native-playtorch.svg?style=flat-square)](https://npmjs.org/package/react-native-playtorch "View this project on npm") +[![npm downloads](http://img.shields.io/npm/dm/react-native-playtorch.svg?style=flat-square)](https://npmjs.org/package/react-native-playtorch "View this project on npm") +[![npm licence](http://img.shields.io/npm/l/react-native-playtorch.svg?style=flat-square)](https://npmjs.org/package/react-native-playtorch "View this project on npm") +[![Platform](https://img.shields.io/badge/platform-ios%20%7C%20android-989898.svg?style=flat-square)](https://npmjs.org/package/react-native-playtorch "View this project on npm") + The PyTorch core library for React Native is part of the PlayTorch project. Please follow the instructions provided on the [PlayTorch website](https://playtorch.dev/) as outlined below! ## 🎉 Getting started with PlayTorch @@ -13,6 +18,77 @@ Follow the [Getting Started guide](https://playtorch.dev/docs/tutorials/get-star The full documentation for PlayTorch can be found on our [website](https://playtorch.dev/). +## Install +``` +npm install react-native-playtorch +``` + +Modify `android/app/build.gradle`: +``` +android { + .... + packagingOptions { + // doNotStrip "**/libc++_shared.so" + pickFirst '**/*.so' + } + ... +} +``` +Modify `android/gradle.properties`: +``` +org.gradle.jvmargs=-Xmx4g +``` +Modify `metro.config.js`: +``` +const defaultAssetExts = require('metro-config/src/defaults/defaults') + .assetExts; + +module.exports = { + resolver: { + assetExts: [...defaultAssetExts, 'ptl'], + }, +}; +``` + +## Patch to fix `__emutls_get_address` crash on Android +If `RN0.71+` and run crash on Android `java.lang.UnsatisfiedLinkError: dlopen failed: cannot locate symbol "__emutls_get_address" referenced by "/data/app/~~Bu6UWdRieDpDrpvvyvNNVQ==/com.foo.bar-w8nusksLnLfSCCsWG3cEkg==/lib/arm64/libfolly_runtime.so"`, you need (e.g. on Linux) +``` +cd tools/android-sdk/ndk + +mv ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/libc++_shared.so ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/origin_libc++_shared.so +cp ./23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/libc++_shared.so ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/ + +mv ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/arm-linux-androideabi/libc++_shared.so ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/arm-linux-androideabi/origin_libc++_shared.so +cp ./23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/arm-linux-androideabi/libc++_shared.so ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/arm-linux-androideabi/ + +mv ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/libc++_shared.so ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/origin_libc++_shared.so +cp ./23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/libc++_shared.so ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/x86_64-linux-android/ + +mv ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/i686-linux-android/libc++_shared.so ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/i686-linux-android/origin_libc++_shared.so +cp ./23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/i686-linux-android/libc++_shared.so ./21.4.7075529/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/i686-linux-android/ +``` +The crash reason is, even the `libc++_shared.so` in `prefab` +``` +readelf -s ~/.gradle/caches/transforms-3/9915f55612e7d9d2d8676faa1872c696/transformed/jetified-react-android-0.71.6-debug/jni/arm64-v8a/libc++_shared.so | grep __emutls_get_address + 374: 00000000000ec5bc 448 FUNC WEAK DEFAULT 16 __emutls_get_address +``` +is `WEAK` not `LOCAL`, but with `pickFirst '**/*.so'`, the `libc++_shared.so` in `.apk` will be picked from `node_modules/react-native-playtorch/android/build/intermediates/library_jni/debug/jni/arm64-v8a/libc++_shared.so`, and +``` +readelf -s node_modules/react-native-playtorch/android/build/intermediates/library_jni/debug/jni/arm64-v8a/libc++_shared.so | grep __emutls_get_address + 3885: 00000000000b60a0 344 FUNC LOCAL DEFAULT 11 __emutls_get_address +``` +is `LOCAL` not `WEAK`. + +If enable `doNotStrip "**/libc++_shared.so"` then extract the `libc++_shared.so` from `.apk`, use `readelf` you will also find it's `LOCAL` not `WEAK`. + +The `libfolly_runtime.so` will call `__emutls_get_address`, if it's `LOCAL`, then run into crash. + +The `__emutls_get_address` in `libc++_shared.so` of `NDK21.4.7075529` is `LOCAL`, and it's `WEAK` for `NDK23.1.7779620`. + +For now, `react-native-playtorch` only can be compiled in `NDK21.4.7075529`. + +So comes the patch above. + ## Example Usage ```javascript @@ -28,7 +104,7 @@ import { Tensor, torch, torchvision, -} from 'react-native-pytorch-core'; +} from 'react-native-playtorch'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; // Alias for torchvision transforms diff --git a/react-native-pytorch-core/android/CMakeLists.txt b/react-native-pytorch-core/android/CMakeLists.txt index 1ddb16180..86f44060d 100644 --- a/react-native-pytorch-core/android/CMakeLists.txt +++ b/react-native-pytorch-core/android/CMakeLists.txt @@ -3,17 +3,31 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +project(PlayTorch) cmake_minimum_required(VERSION 3.4.1) set (CMAKE_VERBOSE_MAKEFILE ON) # This should be matched with the Unit Test Build. Please keep them in sync. See ../cxx/test/CMakeLists.txt set (CMAKE_CXX_STANDARD 14) -set (CMAKE_CXX_FLAGS "-DFOLLY_NO_CONFIG=1 -DFOLLY_HAVE_CLOCK_GETTIME=1 -DFOLLY_HAVE_MEMRCHR=1 -DFOLLY_USE_LIBCPP=1 -DFOLLY_MOBILE=1 -DON_ANDROID -DONANDROID") + +if(${REACT_NATIVE_MINOR_VERSION} GREATER_EQUAL 71) + include("${NODE_MODULES_DIR}/react-native/ReactAndroid/cmake-utils/folly-flags.cmake") + add_compile_options(${folly_FLAGS}) +else() + set (CMAKE_CXX_FLAGS "-DFOLLY_NO_CONFIG=1 -DFOLLY_HAVE_CLOCK_GETTIME=1 -DFOLLY_HAVE_MEMRCHR=1 -DFOLLY_USE_LIBCPP=1 -DFOLLY_MOBILE=1 -DON_ANDROID -DONANDROID") +endif() set (PACKAGE_NAME "torchlive") set (BUILD_DIR ${CMAKE_SOURCE_DIR}/build) -set (RN_SO_DIR ${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/first-party/react/jni) + +if(${REACT_NATIVE_MINOR_VERSION} GREATER_EQUAL 71) + # Consume shared libraries and headers from prefabs + find_package(fbjni REQUIRED CONFIG) + find_package(ReactAndroid REQUIRED CONFIG) +else() + set (RN_SO_DIR ${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/first-party/react/jni) +endif() # PyTorch Core shared @@ -60,25 +74,43 @@ add_library( # includes -file (GLOB LIBFBJNI_INCLUDE_DIR "${BUILD_DIR}/fbjni-*-headers.jar/") file (GLOB PYTORCH_INCLUDE_DIRS "${BUILD_DIR}/pytorch_android_lite*.aar/headers") -target_include_directories( - ${PACKAGE_NAME} - PRIVATE - "${LIBFBJNI_INCLUDE_DIR}" - "${NODE_MODULES_DIR}/react-native/React" - "${NODE_MODULES_DIR}/react-native/React/Base" - "${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni" - "${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/java/com/facebook/react/turbomodule/core/jni" - "${NODE_MODULES_DIR}/react-native/ReactCommon" - "${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker" - "${NODE_MODULES_DIR}/react-native/ReactCommon/jsi" - "${NODE_MODULES_DIR}/react-native/ReactCommon/runtimeexecutor" - "${PYTORCH_INCLUDE_DIRS}" - "../cxx/src" - "src/main/cpp" -) +if(${REACT_NATIVE_MINOR_VERSION} GREATER_EQUAL 71) + target_include_directories( + ${PACKAGE_NAME} + PRIVATE + "${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/react/turbomodule" + "${NODE_MODULES_DIR}/react-native/ReactCommon" + "${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker" + "${NODE_MODULES_DIR}/react-native/ReactCommon/jsi" + "${NODE_MODULES_DIR}/react-native/ReactCommon/react/renderer/graphics/platform/cxx" + "${NODE_MODULES_DIR}/react-native/ReactCommon/runtimeexecutor" + "${NODE_MODULES_DIR}/react-native/ReactCommon/yoga" + "${PYTORCH_INCLUDE_DIRS}" + "../cxx/src" + "src/main/cpp" + ) +else() + file (GLOB LIBFBJNI_INCLUDE_DIR "${BUILD_DIR}/fbjni-*-headers.jar/") + + target_include_directories( + ${PACKAGE_NAME} + PRIVATE + "${LIBFBJNI_INCLUDE_DIR}" + "${NODE_MODULES_DIR}/react-native/React" + "${NODE_MODULES_DIR}/react-native/React/Base" + "${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni" + "${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/java/com/facebook/react/turbomodule/core/jni" + "${NODE_MODULES_DIR}/react-native/ReactCommon" + "${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker" + "${NODE_MODULES_DIR}/react-native/ReactCommon/jsi" + "${NODE_MODULES_DIR}/react-native/ReactCommon/runtimeexecutor" + "${PYTORCH_INCLUDE_DIRS}" + "../cxx/src" + "src/main/cpp" + ) +endif() # find libraries @@ -90,20 +122,21 @@ find_library( log ) -find_library( - FBJNI_LIBRARY - fbjni - PATHS ${LIBRN_DIR} - NO_CMAKE_FIND_ROOT_PATH -) - -find_library( - REACT_NATIVE_JNI_LIB - reactnativejni - PATHS ${LIBRN_DIR} - NO_CMAKE_FIND_ROOT_PATH -) +if(${REACT_NATIVE_MINOR_VERSION} LESS 71) + find_library( + FBJNI_LIBRARY + fbjni + PATHS ${LIBRN_DIR} + NO_CMAKE_FIND_ROOT_PATH + ) + find_library( + REACT_NATIVE_JNI_LIB + reactnativejni + PATHS ${LIBRN_DIR} + NO_CMAKE_FIND_ROOT_PATH + ) +endif() find_library( PYTORCH_LIBRARY @@ -112,8 +145,21 @@ find_library( NO_CMAKE_FIND_ROOT_PATH ) -# For RN 0.66+, use the distributed libjsi.so -if(${REACT_NATIVE_MINOR_VERSION} GREATER_EQUAL 66) +if(${REACT_NATIVE_MINOR_VERSION} GREATER_EQUAL 71) + target_link_libraries( + ${PACKAGE_NAME} + ReactAndroid::folly_runtime + ReactAndroid::glog + ReactAndroid::jsi + ReactAndroid::reactnativejni + fbjni::fbjni + ${PYTORCH_LIBRARY} + ) +elseif(${REACT_NATIVE_MINOR_VERSION} LESS 66) + # JSI lib didn't exist on RN 0.65 and before. Simply omit it. + set (JSI_LIB "") +else() + # For RN 0.66+, use the distributed libjsi.so find_library( JSI_LIB jsi diff --git a/react-native-pytorch-core/android/build.gradle b/react-native-pytorch-core/android/build.gradle index c70681a72..fefdf7c4e 100644 --- a/react-native-pytorch-core/android/build.gradle +++ b/react-native-pytorch-core/android/build.gradle @@ -41,6 +41,18 @@ def safeExtGet(prop, fallback) { android { compileSdkVersion safeExtGet('PyTorchCore_compileSdkVersion', Integer.parseInt(appRnMinorVersion) < 68 ? 30 : 31) buildToolsVersion safeExtGet('PyTorchCore_buildToolsVersion', Integer.parseInt(appRnMinorVersion) < 68 ? '30.0.2' : '31.0.0') + + // ndkVersion getExtOrDefault('ndkVersion') + // use below because ndk 22+ version will cause `ld: error: found local symbol '__bss_start'` + // ref to https://github.com/pytorch/pytorch/issues/51020#issuecomment-1600272599 + ndkVersion "21.4.7075529" + + if (Integer.parseInt(appRnMinorVersion) >= 71) { + buildFeatures { + prefab true + } + } + defaultConfig { minSdkVersion safeExtGet('PyTorchCore_minSdkVersion', 21) targetSdkVersion safeExtGet('PyTorchCore_targetSdkVersion', 30) @@ -67,12 +79,15 @@ android { lintOptions { disable 'GradleCompatible' } - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_8 - targetCompatibility JavaVersion.VERSION_1_8 + if (Integer.parseInt(appRnMinorVersion) < 71) { + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } } externalNativeBuild { cmake { + // version "3.22.1" path "CMakeLists.txt" } } @@ -106,31 +121,43 @@ repositories { dependencies { def pytorchLiteVersion = '1.12.2' - implementation "org.pytorch:pytorch_android_lite:${pytorchLiteVersion}" - - //noinspection GradleDynamicVersion - api "com.facebook.react:react-native:+" // From node_modules - - //noinspection GradleDynamicVersion - extractHeaders("com.facebook.fbjni:fbjni:+:headers") - //noinspection GradleDynamicVersion - extractJNI("com.facebook.fbjni:fbjni:+") - - def nodeModules = "${rootDir}/../node_modules" - - // For now we use the release version of the jni library - // we will look back on this once we figure out how to - // get the buildType duing configuration stage - def buildType = "release" + if (Integer.parseInt(appRnMinorVersion) < 71) { + implementation "org.pytorch:pytorch_android_lite:${pytorchLiteVersion}" + } else { + implementation ("org.pytorch:pytorch_android_lite:${pytorchLiteVersion}") { + exclude group:'com.facebook.fbjni', module: 'fbjni-java-only' + } + } - if (Integer.parseInt(appRnMinorVersion) < 69) { - def rnAAR = fileTree("${nodeModules}/react-native/android").matching({ it.include "**/**/*.aar" }).singleFile - extractJNI(files(rnAAR)) + if (Integer.parseInt(appRnMinorVersion) < 71) { + api "com.facebook.react:react-native:+" // From node_modules } else { - // React Native 0.69 - def rnAarMatcher = "**/react-native/**/*${buildType}.aar" - def rnAAR = fileTree("${nodeModules}/react-native/android").matching({ it.include rnAarMatcher }).singleFile - extractJNI(files(rnAAR)) + //noinspection GradleDynamicVersion + implementation("com.facebook.react:react-android") + } + + if (Integer.parseInt(appRnMinorVersion) < 71) { + //noinspection GradleDynamicVersion + extractHeaders("com.facebook.fbjni:fbjni:+:headers") + //noinspection GradleDynamicVersion + extractJNI("com.facebook.fbjni:fbjni:+") + + def nodeModules = "${rootDir}/../node_modules" + + // For now we use the release version of the jni library + // we will look back on this once we figure out how to + // get the buildType duing configuration stage + def buildType = "release" + + if (Integer.parseInt(appRnMinorVersion) < 69) { + def rnAAR = fileTree("${nodeModules}/react-native/android").matching({ it.include "**/**/*.aar" }).singleFile + extractJNI(files(rnAAR)) + } else if (Integer.parseInt(appRnMinorVersion) < 71) { + // React Native 0.69 + def rnAarMatcher = "**/react-native/**/*${buildType}.aar" + def rnAAR = fileTree("${nodeModules}/react-native/android").matching({ it.include rnAarMatcher }).singleFile + extractJNI(files(rnAAR)) + } } extractForNativeBuild("org.pytorch:pytorch_android_lite:${pytorchLiteVersion}") @@ -207,8 +234,8 @@ task extractAARForNativeBuild { def configureCMakeTaskName = Integer.parseInt(appRnMinorVersion) < 68 ? "externalNativeBuild" : "configureCMake" tasks.whenTaskAdded { task -> if (task.name.contains(configureCMakeTaskName)) { - task.dependsOn(extractAARHeaders) task.dependsOn(extractJNIFiles) + task.dependsOn(extractAARHeaders) task.dependsOn(extractAARForNativeBuild) } } diff --git a/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/PyTorchCorePackage.java b/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/PyTorchCorePackage.java index 325b427c5..ef39dd2d4 100644 --- a/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/PyTorchCorePackage.java +++ b/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/PyTorchCorePackage.java @@ -15,6 +15,7 @@ import java.util.List; import org.jetbrains.annotations.NotNull; import org.pytorch.rn.core.audio.AudioModule; +import org.pytorch.rn.core.camera.CameraModule; import org.pytorch.rn.core.camera.CameraViewManager; import org.pytorch.rn.core.canvas.CanvasRenderingContext2DModule; import org.pytorch.rn.core.canvas.CanvasViewManager; @@ -29,6 +30,7 @@ public class PyTorchCorePackage implements ReactPackage { public List createNativeModules(@NotNull ReactApplicationContext reactContext) { return Arrays.asList( new ModelLoaderModule(reactContext), + new CameraModule(reactContext), new ImageModule(reactContext), new ImageDataModule(reactContext), new CanvasRenderingContext2DModule(reactContext), diff --git a/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/camera/CameraModule.java b/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/camera/CameraModule.java new file mode 100644 index 000000000..2902c429a --- /dev/null +++ b/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/camera/CameraModule.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +package org.pytorch.rn.core.camera; + +import android.app.Activity; +import android.util.Log; +import android.view.View; +import androidx.annotation.NonNull; + +import com.facebook.react.bridge.Promise; +import com.facebook.react.bridge.ReactApplicationContext; +import com.facebook.react.bridge.ReactContextBaseJavaModule; +import com.facebook.react.bridge.ReactMethod; +import com.facebook.react.module.annotations.ReactModule; + +@ReactModule(name = "PyTorchCoreCameraModule") +public class CameraModule extends ReactContextBaseJavaModule { + + private static final String TAG = CameraModule.class.getSimpleName(); + + private static final String NAME = "PyTorchCoreCameraModule"; + + public CameraModule(ReactApplicationContext reactContext) { + super(reactContext); + } + + @NonNull + @Override + public String getName() { + return NAME; + } + + @ReactMethod + public void takePicture(int reactTag, boolean isPreviewView, Promise promise) { + Activity activity = getCurrentActivity(); + View view = activity.findViewById(reactTag); + + try { + CameraView cameraView = (CameraView) view; + cameraView.takePicture(activity, isPreviewView, promise); + } catch (Exception e) { + // Sometimes(when RELOAD js of react-native) cause + // `java.lang.ClassCastException: com.facebook.react.views.view.ReactViewGroup cannot be cast to org.pytorch.rn.core.camera.CameraView` + // and crash the APP, then found catch Exception + // and just return here is OK. + + // System.out.println(e); + Log.e(TAG, Log.getStackTraceString(e)); + return; + } + } +} diff --git a/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/camera/CameraView.java b/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/camera/CameraView.java index 3e0ae0c26..4c53a70e4 100644 --- a/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/camera/CameraView.java +++ b/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/camera/CameraView.java @@ -9,6 +9,7 @@ import android.Manifest; import android.animation.ValueAnimator; +import android.app.Activity; import android.content.pm.PackageManager; import android.graphics.Color; import android.graphics.drawable.GradientDrawable; @@ -36,6 +37,7 @@ import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import androidx.lifecycle.LifecycleOwner; +import com.facebook.react.bridge.Promise; import com.facebook.react.bridge.ReactApplicationContext; import com.facebook.react.uimanager.events.RCTEventEmitter; import com.google.common.util.concurrent.ListenableFuture; @@ -215,6 +217,40 @@ public void onError(@NonNull ImageCaptureException exception) { } } + protected void takePicture(Activity activity, boolean isPreviewView, Promise promise) { + if (mImageCapture != null) { + if (isPreviewView) { + activity.runOnUiThread(new Runnable() { + @Override + public void run() { + IImage image = new Image(mPreviewView.getBitmap()); + JSContext.NativeJSRef ref = JSContext.wrapObject(image); + promise.resolve(ref.getJSRef()); + } + }); + } else { + mImageCapture.takePicture( + ContextCompat.getMainExecutor(mReactContext), + new ImageCapture.OnImageCapturedCallback() { + @Override + public void onCaptureSuccess(@NonNull ImageProxy imageProxy) { + super.onCaptureSuccess(imageProxy); + IImage image = new Image(imageProxy, mReactContext.getApplicationContext()); + JSContext.NativeJSRef ref = JSContext.wrapObject(image); + promise.resolve(ref.getJSRef()); + } + + @Override + public void onError(@NonNull ImageCaptureException exception) { + super.onError(exception); + Log.e(TAG, exception.getLocalizedMessage(), exception); + promise.reject(exception); + } + }); + } + } + } + protected void flipCamera() { if (mPreferredCameraSelector == CameraSelector.DEFAULT_BACK_CAMERA) { mPreferredCameraSelector = CameraSelector.DEFAULT_FRONT_CAMERA; diff --git a/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/image/ImageModule.java b/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/image/ImageModule.java index 34887a487..6c3abb83f 100644 --- a/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/image/ImageModule.java +++ b/react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/image/ImageModule.java @@ -12,11 +12,13 @@ import android.net.Uri; import android.widget.Toast; import androidx.annotation.NonNull; +import com.facebook.react.bridge.Arguments; import com.facebook.react.bridge.Promise; import com.facebook.react.bridge.ReactApplicationContext; import com.facebook.react.bridge.ReactContextBaseJavaModule; import com.facebook.react.bridge.ReactMethod; import com.facebook.react.bridge.ReadableMap; +import com.facebook.react.bridge.WritableArray; import com.facebook.react.common.build.ReactBuildConfig; import com.facebook.react.module.annotations.ReactModule; import com.facebook.react.views.imagehelper.ImageSource; @@ -27,6 +29,7 @@ import java.io.InputStream; import java.net.URL; import org.pytorch.rn.core.canvas.ImageData; +import org.pytorch.rn.core.image.ImageUtils; import org.pytorch.rn.core.javascript.JSContext; import org.pytorch.rn.core.utils.FileUtils; @@ -187,6 +190,18 @@ public void fromImageData(final ReadableMap imageDataRef, final boolean scaled, promise.resolve(ref.getJSRef()); } + @ReactMethod + public void toImageDataDataWeb(final ReadableMap imageRef, Promise promise) { + IImage image = JSContext.unwrapObject(imageRef); + byte[] bytes = ImageUtils.bitmapToRGBA(image.getBitmap()); + + WritableArray array = Arguments.createArray(); + for (int i = 0; i < bytes.length; i++) { + array.pushInt(bytes[i]); + } + promise.resolve(array); + } + @ReactMethod public void toFile(final ReadableMap imageRef, Promise promise) { try { diff --git a/react-native-pytorch-core/ios/Canvas/CanvasRenderingContext2D.swift b/react-native-pytorch-core/ios/Canvas/CanvasRenderingContext2D.swift index 90dc40221..badd4268f 100644 --- a/react-native-pytorch-core/ios/Canvas/CanvasRenderingContext2D.swift +++ b/react-native-pytorch-core/ios/Canvas/CanvasRenderingContext2D.swift @@ -519,7 +519,7 @@ class CanvasRenderingContext2D: NSObject { rejecter reject: RCTPromiseRejectBlock) { do { let canvasView = try JSContextUtils.unwrapObject(canvasRef, DrawingCanvasView.self) - let image = try JSContextUtils.unwrapObject(image, IImage.self) + let image = try JSContextUtils.unwrapObject(image, Image.self) if dWidth == -1 && sWidth == -1 { try canvasView.drawImage(image: image, dx: CGFloat(truncating: sx), dy: CGFloat(truncating: sy)) } else if dx == -1 { diff --git a/react-native-pytorch-core/ios/Canvas/DrawingCanvasView.swift b/react-native-pytorch-core/ios/Canvas/DrawingCanvasView.swift index c03f0997b..95c351f85 100644 --- a/react-native-pytorch-core/ios/Canvas/DrawingCanvasView.swift +++ b/react-native-pytorch-core/ios/Canvas/DrawingCanvasView.swift @@ -386,7 +386,7 @@ class DrawingCanvasView: UIView { sublayers.append(newLayer) } - func drawImage(image: IImage, dx: CGFloat, dy: CGFloat) throws { + func drawImage(image: Image, dx: CGFloat, dy: CGFloat) throws { let frame = CGRect(x: dx, y: dy, width: CGFloat(image.getWidth()), height: CGFloat(image.getHeight())) if let bitmap = image.getBitmap() { let newLayer = ImageLayerData(image: bitmap, transform: currentState.transform, frame: frame) @@ -396,7 +396,7 @@ class DrawingCanvasView: UIView { } } - func drawImage(image: IImage, dx: CGFloat, dy: CGFloat, dWidth: CGFloat, dHeight: CGFloat) throws { + func drawImage(image: Image, dx: CGFloat, dy: CGFloat, dWidth: CGFloat, dHeight: CGFloat) throws { let frame = CGRect(x: dx, y: dy, width: dWidth, height: dHeight) if let bitmap = image.getBitmap() { let newLayer = ImageLayerData(image: bitmap, transform: currentState.transform, frame: frame) @@ -406,7 +406,7 @@ class DrawingCanvasView: UIView { } } - func drawImage(image: IImage, + func drawImage(image: Image, sx: CGFloat, sy: CGFloat, sWidth: CGFloat, diff --git a/react-native-pytorch-core/ios/Image/ImageModule.swift b/react-native-pytorch-core/ios/Image/ImageModule.swift index d4601df36..da27206af 100644 --- a/react-native-pytorch-core/ios/Image/ImageModule.swift +++ b/react-native-pytorch-core/ios/Image/ImageModule.swift @@ -77,7 +77,7 @@ public class ImageModule: NSObject { do { let imageData = try JSContextUtils.unwrapObject(imageDataRef, ImageData.self) - var image: IImage + var image: Image if scaled { let bitmap = try imageData.getScaledBitmap() image = Image(image: bitmap) @@ -99,7 +99,7 @@ public class ImageModule: NSObject { resolver resolve: RCTPromiseResolveBlock, rejecter reject: RCTPromiseRejectBlock) { do { - let image = try JSContextUtils.unwrapObject(imageRef, IImage.self) + let image = try JSContextUtils.unwrapObject(imageRef, Image.self) guard let bitmap = image.getBitmap() else { print("Could not get bitmap from image") return @@ -125,7 +125,7 @@ public class ImageModule: NSObject { @objc public func getWidth(_ imageRef: NSDictionary) -> Any { do { - let image = try JSContextUtils.unwrapObject(imageRef, IImage.self) + let image = try JSContextUtils.unwrapObject(imageRef, Image.self) return NSNumber(value: Float(image.getWidth())) } catch { print("Invalid image reference in getWidth") @@ -136,7 +136,7 @@ public class ImageModule: NSObject { @objc public func getHeight(_ imageRef: NSDictionary) -> Any { do { - let image = try JSContextUtils.unwrapObject(imageRef, IImage.self) + let image = try JSContextUtils.unwrapObject(imageRef, Image.self) return NSNumber(value: Float(image.getHeight())) } catch { print("Invalid image reference in getHeight") @@ -151,7 +151,7 @@ public class ImageModule: NSObject { resolver resolve: RCTPromiseResolveBlock, rejecter reject: RCTPromiseRejectBlock) { do { - let image = try JSContextUtils.unwrapObject(imageRef, IImage.self) + let image = try JSContextUtils.unwrapObject(imageRef, Image.self) let scaledImage = try image.scale(sx: CGFloat(truncating: sx), sy: CGFloat(truncating: sy)) let ref = JSContext.wrapObject(object: scaledImage).getJSRef() resolve(ref) diff --git a/react-native-pytorch-core/ios/Media/MediaToBlob.swift b/react-native-pytorch-core/ios/Media/MediaToBlob.swift index 147873131..a3bb359cc 100644 --- a/react-native-pytorch-core/ios/Media/MediaToBlob.swift +++ b/react-native-pytorch-core/ios/Media/MediaToBlob.swift @@ -26,9 +26,9 @@ public class MediaToBlob: NSObject { public init(refId: String) throws { super.init() let obj = try JSContext.unwrapObject(jsRef: ["ID": refId]) - if obj is IImage { + if obj is Image { // swiftlint:disable:next force_cast - let image = obj as! IImage + let image = obj as! Image imageToBlob(img: image) } else if obj is IAudio { // swiftlint:disable:next force_cast @@ -39,7 +39,7 @@ public class MediaToBlob: NSObject { } } - private func imageToBlob(img: IImage) { + private func imageToBlob(img: Image) { let bitmap = img.getBitmap()! let width = bitmap.width let height = bitmap.height diff --git a/react-native-pytorch-core/ios/PyTorchCore-Swift-Header.h b/react-native-pytorch-core/ios/PyTorchCore-Swift-Header.h index e173e40d4..46d571382 100644 --- a/react-native-pytorch-core/ios/PyTorchCore-Swift-Header.h +++ b/react-native-pytorch-core/ios/PyTorchCore-Swift-Header.h @@ -9,4 +9,4 @@ // Imports needed to call Swift from Objective-C // #import -#import "react_native_pytorch_core-Swift.h" +#import "react_native_playtorch-Swift.h" diff --git a/react-native-pytorch-core/package.json b/react-native-pytorch-core/package.json index 7ff0bf848..b1b7fa0ba 100644 --- a/react-native-pytorch-core/package.json +++ b/react-native-pytorch-core/package.json @@ -1,7 +1,8 @@ { - "name": "react-native-pytorch-core", - "version": "0.0.0", + "name": "react-native-playtorch", + "version": "1.0.6", "description": "PyTorch core library for React Native", + "private": false, "main": "lib/commonjs/index", "module": "lib/module/index", "types": "lib/typescript/index.d.ts", @@ -13,7 +14,7 @@ "android", "ios", "cxx", - "react-native-pytorch-core.podspec", + "react-native-playtorch.podspec", "!lib/typescript/example", "!android/build", "!ios/build", @@ -36,19 +37,25 @@ "doc": "typedoc --plugin typedoc-plugin-markdown src/index.tsx" }, "keywords": [ - "react-native", + "android", "ios", - "android" + "playtorch", + "pytorch", + "tensor", + "torch", + "torchlive", + "torchvision", + "react-native" ], "repository": { "type": "git", - "url": "https://github.com/facebookresearch/playtorch.git", + "url": "https://github.com/flyskywhy/playtorch.git", "directory": "react-native-pytorch-core" }, "author": "Meta Platforms, Inc.", "license": "MIT", "bugs": { - "url": "https://github.com/facebookresearch/playtorch/issues" + "url": "https://github.com/flyskywhy/playtorch/issues" }, "homepage": "https://playtorch.dev/", "publishConfig": { diff --git a/react-native-pytorch-core/react-native-pytorch-core.podspec b/react-native-pytorch-core/react-native-playtorch.podspec similarity index 90% rename from react-native-pytorch-core/react-native-pytorch-core.podspec rename to react-native-pytorch-core/react-native-playtorch.podspec index 30ed1bd0e..47bfe0c66 100644 --- a/react-native-pytorch-core/react-native-pytorch-core.podspec +++ b/react-native-pytorch-core/react-native-playtorch.podspec @@ -8,7 +8,7 @@ require "json" package = JSON.parse(File.read(File.join(__dir__, "package.json"))) Pod::Spec.new do |s| - s.name = "react-native-pytorch-core" + s.name = "react-native-playtorch" s.version = package["version"] s.summary = package["description"] s.homepage = package["homepage"] @@ -16,7 +16,7 @@ Pod::Spec.new do |s| s.authors = package["author"] s.platforms = { :ios => "12.0" } - s.source = { :git => "https://github.com/facebookresearch/playtorch.git", :tag => "#{s.version}" } + s.source = { :git => "https://github.com/flyskywhy/playtorch.git", :tag => "#{s.version}" } s.source_files = "ios/**/*.{h,m,mm,swift,cpp}", "cxx/src/**/*.{h,cpp}" diff --git a/react-native-pytorch-core/src/CameraView.tsx b/react-native-pytorch-core/src/CameraView.tsx index 86a737955..cadb0ffc2 100644 --- a/react-native-pytorch-core/src/CameraView.tsx +++ b/react-native-pytorch-core/src/CameraView.tsx @@ -11,6 +11,8 @@ import * as React from 'react'; import { findNodeHandle, requireNativeComponent, + NativeModules, + Platform, UIManager, ViewProps, } from 'react-native'; @@ -116,6 +118,10 @@ const nativeCameraViewName = 'PyTorchCoreCameraView'; const PyTorchCoreCameraView = requireNativeComponent(nativeCameraViewName); +const nativeCameraModuleName = 'PyTorchCoreCameraModule'; + +const PyTorchCoreCameraModule = NativeModules[nativeCameraModuleName]; + /** * A camera component with [[CameraProps.onCapture]] and [[CameraProps.onFrame]] callbacks. * To programmatically trigger a capture, call the [[takePicture]] function. @@ -195,17 +201,31 @@ export class Camera extends React.PureComponent { * } * ``` */ - public takePicture(): void { + // if isPreviewView is true, will use preview instead of photo, + // this can reduce 400ms to 100ms, but since preview will delay + // some from real target, maybe it's nonsense, so default false + public async takePicture(isPreviewView: boolean = false): Promise { if (this.cameraRef.current) { - const takePictureCommandId = - UIManager.getViewManagerConfig(nativeCameraViewName).Commands - .takePicture; const cameraViewHandle = findNodeHandle(this.cameraRef.current); - UIManager.dispatchViewManagerCommand( - cameraViewHandle, - takePictureCommandId, - [], - ); + if (Platform.OS === 'android') { + // TODO: also implement ios + const nativeEvent = await PyTorchCoreCameraModule.takePicture( + cameraViewHandle, + isPreviewView, + ); + if (nativeEvent.ID) { + this.handleOnCapture({nativeEvent}); + } + } else { + const takePictureCommandId = + UIManager.getViewManagerConfig(nativeCameraViewName).Commands + .takePicture; + UIManager.dispatchViewManagerCommand( + cameraViewHandle, + takePictureCommandId, + [], + ); + } } } diff --git a/react-native-pytorch-core/src/ImageModule.ts b/react-native-pytorch-core/src/ImageModule.ts index 31d1db2e2..e1d7283ab 100644 --- a/react-native-pytorch-core/src/ImageModule.ts +++ b/react-native-pytorch-core/src/ImageModule.ts @@ -186,6 +186,11 @@ export const ImageUtil = { return wrapRef(ref); }, + // 2073600 bytes of 720x720 cost 5 seconds to await + async toImageDataDataWeb(image: Image): Promise { + return await ImageModule.toImageDataDataWeb(image); + }, + /** * The `fromJSRef` function returns an [[Image]] by wrapping a [[NativeJSRef]] * object. diff --git a/react-native-pytorch-core/src/expo-plugin/withPyTorchCore.ts b/react-native-pytorch-core/src/expo-plugin/withPyTorchCore.ts index d023f69a6..0f9a68623 100644 --- a/react-native-pytorch-core/src/expo-plugin/withPyTorchCore.ts +++ b/react-native-pytorch-core/src/expo-plugin/withPyTorchCore.ts @@ -15,7 +15,7 @@ import { } from '@expo/config-plugins'; // Keeping the name, and version in sync with it's package. -const pkg = require('react-native-pytorch-core/package.json'); +const pkg = require('react-native-playtorch/package.json'); type Props = {}; @@ -93,7 +93,7 @@ tasks.whenTaskAdded { task -> dependencies { // Used to control the version of libfbjni.so packaged into the APK - extraJNILibs("com.facebook.fbjni:fbjni:0.2.2") + extraJNILibs("com.facebook.fbjni:fbjni:0.3.0") `, ); diff --git a/react-native-pytorch-core/src/torchlive/media.ts b/react-native-pytorch-core/src/torchlive/media.ts index d5fae2b07..28ca2504a 100644 --- a/react-native-pytorch-core/src/torchlive/media.ts +++ b/react-native-pytorch-core/src/torchlive/media.ts @@ -7,7 +7,7 @@ * @format */ -import type {Tensor} from 'react-native-pytorch-core'; +import type {Tensor} from 'react-native-playtorch'; import type {NativeJSRef} from '../NativeJSRef'; import type {Image} from '../ImageModule'; diff --git a/react-native-pytorch-core/tsconfig.json b/react-native-pytorch-core/tsconfig.json index b3db08f34..b1c6b586f 100644 --- a/react-native-pytorch-core/tsconfig.json +++ b/react-native-pytorch-core/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "baseUrl": "./", "paths": { - "react-native-pytorch-core": ["./src/index"] + "react-native-playtorch": ["./src/index"] }, "allowUnreachableCode": false, "allowUnusedLabels": false,