From b257959ae2f2c234177cb86ef5a54069ee2d8fb1 Mon Sep 17 00:00:00 2001 From: Wook Song Date: Thu, 26 Sep 2024 16:58:42 +0900 Subject: [PATCH] App/Vision: Add a vision example that the exploits MLAgent Service This patch adds a draft of the vision example (i.e., an object classification scenario using MobileNet_v1) that exploits the MLAgent Service. Signed-off-by: Wook Song --- gradle/libs.versions.toml | 9 ++ ml_inference_offloading/build.gradle.kts | 12 ++ .../src/main/AndroidManifest.xml | 5 + .../ml/inference/offloading/MainActivity.kt | 104 ++++++++++-- .../ml/inference/offloading/MainService.kt | 39 ++++- .../offloading/data/Classification.kt | 3 + .../offloading/data/ImageAnalyzer.kt | 18 +++ .../offloading/domain/MobilenetClassifier.kt | 152 ++++++++++++++++++ .../offloading/domain/ObjectClassifier.kt | 8 + .../inference/offloading/ui/MainViewModel.kt | 2 +- 10 files changed, 341 insertions(+), 11 deletions(-) create mode 100644 ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/Classification.kt create mode 100644 ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/ImageAnalyzer.kt create mode 100644 ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/domain/MobilenetClassifier.kt create mode 100644 ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/domain/ObjectClassifier.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 733c1f7..8680aa6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,6 +29,8 @@ tensorflowLite = "2.16.1" tukaani-xz-plugin = "1.9" uiTestJunit4Android = "1.6.8" xyz-simple-git-plugin = "2.0.3" +cameraCore = "1.3.4" +firebaseCrashlyticsBuildtools = "3.0.2" [libraries] commons-compress = { group = "org.apache.commons", name = "commons-compress", version.ref = "commons-compress-lib" } @@ -43,6 +45,12 @@ robolectric = { module = "org.robolectric:robolectric", version.ref = "robolectr tukaani-xz = { group = "org.tukaani", name = "xz", version.ref = "tukaani-xz-plugin" } androidx-activity-compose = { group = "androidx.activity", name = "activity-compose", version.ref = "activityCompose" } androidx-appcompat = { group = "androidx.appcompat", name = "appcompat", version.ref = "appcompat" } +androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "cameraCore" } +androidx-camera-core = { module = "androidx.camera:camera-core", version.ref = "cameraCore" } +androidx-camera-extensions = { module = "androidx.camera:camera-extensions", version.ref = "cameraCore" } +androidx-camera-lifecycle = { module = "androidx.camera:camera-lifecycle", version.ref = "cameraCore" } +androidx-camera-video = { module = "androidx.camera:camera-video", version.ref = "cameraCore" } +androidx-camera-view = { module = "androidx.camera:camera-view", version.ref = "cameraCore" } androidx-compose-bom = { group = "androidx.compose", name = "compose-bom", version.ref = "composeBom" } androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } androidx-datastore-core-android = { group = "androidx.datastore", name = "datastore-core-android", version.ref = "datastore" } @@ -68,6 +76,7 @@ androidx-ui-test-manifest = { group = "androidx.compose.ui", name = "ui-test-man androidx-ui-test-junit4 = { group = "androidx.compose.ui", name = "ui-test-junit4" } androidx-ui-test-junit4-android = { group = "androidx.compose.ui", name = "ui-test-junit4-android", version.ref = "uiTestJunit4Android" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } +firebase-crashlytics-buildtools = { group = "com.google.firebase", name = "firebase-crashlytics-buildtools", version.ref = "firebaseCrashlyticsBuildtools" } [plugins] androidApplication = { id = "com.android.application", version.ref = "agp" } diff --git a/ml_inference_offloading/build.gradle.kts b/ml_inference_offloading/build.gradle.kts index d12cef1..4f46568 100644 --- a/ml_inference_offloading/build.gradle.kts +++ b/ml_inference_offloading/build.gradle.kts @@ -87,6 +87,7 @@ dependencies { implementation(libs.androidx.core.ktx) implementation(libs.androidx.lifecycle.runtime.compose) implementation(libs.androidx.lifecycle.runtime.ktx) + implementation(libs.androidx.lifecycle.runtime.compose) implementation(libs.androidx.material.icons.core) implementation(libs.androidx.material.icons.extended) implementation(libs.androidx.material3) @@ -99,6 +100,17 @@ dependencies { // Room implementation(libs.androidx.room.common) implementation(libs.androidx.room.runtime) + + // Camera + implementation(libs.androidx.camera.view) + implementation(libs.androidx.camera.core) + implementation(libs.androidx.camera.camera2) + implementation(libs.androidx.camera.lifecycle) + implementation(libs.androidx.camera.video) + implementation(libs.androidx.camera.view) + implementation(libs.androidx.camera.extensions) + implementation(libs.firebase.crashlytics.buildtools) + ksp(libs.androidx.room.compiler) implementation(libs.androidx.room.ktx) diff --git a/ml_inference_offloading/src/main/AndroidManifest.xml b/ml_inference_offloading/src/main/AndroidManifest.xml index 8846c1d..91db086 100644 --- a/ml_inference_offloading/src/main/AndroidManifest.xml +++ b/ml_inference_offloading/src/main/AndroidManifest.xml @@ -1,6 +1,11 @@ + + + diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainActivity.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainActivity.kt index 2187536..b5d4fa6 100644 --- a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainActivity.kt +++ b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainActivity.kt @@ -1,25 +1,35 @@ package ai.nnstreamer.ml.inference.offloading +import ai.nnstreamer.ml.inference.offloading.data.Classification +import ai.nnstreamer.ml.inference.offloading.data.ImageAnalyzer +import ai.nnstreamer.ml.inference.offloading.domain.MobilenetClassifier import ai.nnstreamer.ml.inference.offloading.ui.MainViewModel import ai.nnstreamer.ml.inference.offloading.ui.components.ButtonList import ai.nnstreamer.ml.inference.offloading.ui.components.ServiceList import ai.nnstreamer.ml.inference.offloading.ui.theme.NnstreamerandroidTheme +import android.Manifest import android.annotation.SuppressLint import android.content.ComponentName import android.content.Context import android.content.Intent import android.content.ServiceConnection +import android.content.pm.PackageManager import android.os.Bundle import android.os.IBinder import android.os.Message import android.os.Messenger import androidx.activity.ComponentActivity import androidx.activity.compose.setContent +import androidx.camera.view.CameraController +import androidx.camera.view.LifecycleCameraController +import androidx.camera.view.PreviewView +import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding import androidx.compose.material.icons.Icons @@ -44,16 +54,24 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.TopAppBar import androidx.compose.material3.rememberDrawerState -import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableIntStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.core.app.ActivityCompat +import androidx.core.content.ContextCompat +import androidx.lifecycle.compose.LocalLifecycleOwner import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable @@ -112,6 +130,14 @@ class MainActivity : ComponentActivity() { // Dependency Injection (application as App).appComponent.inject(this) super.onCreate(savedInstanceState) + if (!hasCameraPermission()) { + ActivityCompat.requestPermissions( + this, + arrayOf(Manifest.permission.CAMERA), + 0 + ) + } + setContent { NnstreamerandroidTheme { val navController = rememberNavController() @@ -268,17 +294,55 @@ class MainActivity : ComponentActivity() { } composable { - Column( - modifier = Modifier.fillMaxSize(), - verticalArrangement = Arrangement.Center, - horizontalAlignment = Alignment.CenterHorizontally - ) { - Box(modifier = Modifier.fillMaxSize()) { - Text(text = "Vision Examples") + var classifications by remember { + mutableStateOf(emptyList()) + } + val analyzer = remember { + ImageAnalyzer( + MobilenetClassifier( + mService, + onResults = { + classifications = it + }), + ) + } + val controller = remember { + LifecycleCameraController(applicationContext).apply { + setEnabledUseCases(CameraController.IMAGE_ANALYSIS) + setImageAnalysisAnalyzer( + ContextCompat.getMainExecutor( + applicationContext + ), + analyzer + ) } } + Box(modifier = Modifier.fillMaxSize()) { + CameraPreview( + controller, + Modifier + .fillMaxSize() + ) + Column( + modifier = Modifier + .fillMaxSize() + .align(Alignment.BottomCenter), + ) { + classifications.forEach { classification -> + Text( + text = "${classification.label} (${classification.confidence}%)", + modifier = Modifier + .fillMaxWidth() + .background(colorScheme.secondaryContainer) + .padding(16.dp), + textAlign = TextAlign.Center, + fontSize = 20.sp, + color = colorScheme.onSecondaryContainer, + ) + } + } // Column + } // Box } - composable { val args = it.toRoute() Column( @@ -321,6 +385,28 @@ class MainActivity : ComponentActivity() { super.onStop() unbindService(connection) } + + @Composable + fun CameraPreview( + controller: LifecycleCameraController, + modifier: Modifier = Modifier + ) { + val lifecycleOwner = LocalLifecycleOwner.current + + AndroidView( + factory = { + PreviewView(it).apply { + this.controller = controller + controller.bindToLifecycle(lifecycleOwner) + } + }, + modifier = modifier, + ) + } + + private fun hasCameraPermission() = ContextCompat.checkSelfPermission( + this, Manifest.permission.CAMERA + ) == PackageManager.PERMISSION_GRANTED } @Serializable diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainService.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainService.kt index fb9be64..3c866b7 100644 --- a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainService.kt +++ b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/MainService.kt @@ -19,6 +19,7 @@ import android.net.nsd.NsdManager import android.net.nsd.NsdManager.RegistrationListener import android.net.nsd.NsdServiceInfo import android.os.Build +import android.os.Bundle import android.os.Handler import android.os.HandlerThread import android.os.IBinder @@ -46,7 +47,8 @@ enum class MessageType(val value: Int) { LOAD_MODELS(0), START_MODEL(1), STOP_MODEL(2), - DESTROY_MODEL(3) + DESTROY_MODEL(3), + REQ_OBJ_CLASSIFICATION_FILTER(4), } /** @@ -81,6 +83,41 @@ class MainService : Service() { MessageType.DESTROY_MODEL.value -> destroyService(msg.arg1) + // todo: Generalize the message handling for ML service requests + MessageType.REQ_OBJ_CLASSIFICATION_FILTER.value -> { + val models = modelsRepository.getAllModelsStream() + val bundle = Bundle() + val replyMsg = Message() + val labels = mutableListOf() + + // todo: Remove hardcoded label file path + val imagenetLabelPath = + applicationContext.getExternalFilesDir("models")?.run { + resolve("imagenet_labels.txt") + } + + imagenetLabelPath?.bufferedReader()?.use { reader -> + reader.lineSequence().forEach { line -> + labels.add(line) + } + } + + bundle.putStringArray("labels", labels.toTypedArray()) + // fixme: This is dangerous. We should not use blocking calls in handler. + runBlocking { + models.collect { + it.forEach { model -> + if (model.name.contains("mobilenet")) { + bundle.putString("filter", model.getNNSFilterDesc()) + + replyMsg.data = bundle + msg.replyTo.send(replyMsg) + } + } + } + } + } + else -> super.handleMessage(msg) } } diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/Classification.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/Classification.kt new file mode 100644 index 0000000..e34757b --- /dev/null +++ b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/Classification.kt @@ -0,0 +1,3 @@ +package ai.nnstreamer.ml.inference.offloading.data + +data class Classification(val label: String, val confidence: Double) diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/ImageAnalyzer.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/ImageAnalyzer.kt new file mode 100644 index 0000000..4cded88 --- /dev/null +++ b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/data/ImageAnalyzer.kt @@ -0,0 +1,18 @@ +package ai.nnstreamer.ml.inference.offloading.data + +import ai.nnstreamer.ml.inference.offloading.domain.MobilenetClassifier +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy + +class ImageAnalyzer( + private val classifier: MobilenetClassifier, +) : ImageAnalysis.Analyzer { + override fun analyze(imageProxy: ImageProxy) { + val rotationDegrees = imageProxy.imageInfo.rotationDegrees + val bitmap = imageProxy.toBitmap() + + classifier.classify(bitmap, rotationDegrees) + + imageProxy.close() + } +} diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/domain/MobilenetClassifier.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/domain/MobilenetClassifier.kt new file mode 100644 index 0000000..d3db83f --- /dev/null +++ b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/domain/MobilenetClassifier.kt @@ -0,0 +1,152 @@ +package ai.nnstreamer.ml.inference.offloading.domain + +import ai.nnstreamer.ml.inference.offloading.MessageType +import ai.nnstreamer.ml.inference.offloading.data.Classification +import android.graphics.Bitmap +import android.os.Handler +import android.os.Looper +import android.os.Message +import android.os.Messenger +import android.util.Log +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.launch +import kotlinx.coroutines.supervisorScope +import org.nnsuite.nnstreamer.NNStreamer +import org.nnsuite.nnstreamer.Pipeline +import org.nnsuite.nnstreamer.TensorsData +import org.nnsuite.nnstreamer.TensorsInfo +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import kotlin.experimental.and + +object MobileNetPipeline { + private const val TAG = "MobileNetPipeline" + private const val SRC_PAD = "srcx" + private const val SINK_PAD = "sinkx" + var onResultsCb: ((List) -> Unit)? = null + var pipeline: Pipeline? = null + var labels = arrayOf() + + private fun stateChangeCallback(): Pipeline.StateChangeCallback = + Pipeline.StateChangeCallback { state -> + when (state) { + Pipeline.State.UNKNOWN -> { + Log.d(TAG, state.toString()) + } + + Pipeline.State.NULL -> { + Log.d(TAG, state.toString()) + } + + Pipeline.State.READY -> { + Log.d(TAG, state.toString()) + } + + Pipeline.State.PAUSED -> { + Log.d(TAG, state.toString()) + } + + Pipeline.State.PLAYING -> { + Log.d(TAG, state.toString()) + } + + null -> { + Log.e(TAG, "null") + } + } + } + + + fun incomingNewDataCb(): Pipeline.NewDataCallback = Pipeline.NewDataCallback { data -> + var label = "" + var maxScore = 0 + // todo: Use a list/array compression + data.getTensorData(0).run { + var index = -1 + + for (i in 0..1000) { + val score: Int = (this.get(i).and(0xFF.toByte())).toInt() + + if (score > maxScore) { + maxScore = score + index = i + } + } + + label = labels[index] + } + + onResultsCb?.run { + val classifications = mutableListOf() + + classifications.add(Classification(label, maxScore.toDouble())) + this(classifications) + } + } + + fun create(filter: String) { + val desc = + "appsrc caps=image/jpeg name=$SRC_PAD ! jpegdec ! videoconvert ! videoscale ! tensor_converter ! $filter ! tensor_sink name=$SINK_PAD" + pipeline = Pipeline(desc, stateChangeCallback()).apply { + registerSinkCallback( + SINK_PAD, + incomingNewDataCb() + ) + start() + } + } + + var state = pipeline?.getState() + val pushInputData: (inData: TensorsData) -> Unit = { inData -> + pipeline?.inputData(SRC_PAD, inData) + } +} + +class MobilenetClassifier( + messenger: Messenger?, + onResults: (List) -> Unit +) : ObjectClassifier { + init { + MobileNetPipeline.onResultsCb = onResults + if (MobileNetPipeline.pipeline == null) { + val msg = Message.obtain(null, MessageType.REQ_OBJ_CLASSIFICATION_FILTER.value) + val callback = Handler.Callback { + CoroutineScope(Dispatchers.Main.immediate).launch { + supervisorScope { + val filter = async { + it.data.getString("filter") ?: "" + }.await() + MobileNetPipeline.create(filter) + val labels = async { it.data.getStringArray("labels") }.await() + if (labels != null) { + MobileNetPipeline.labels = labels + } + } // supervisorScope + } // CoroutineScope + true + } // Handler.Callback + msg.replyTo = Messenger(Handler(Looper.getMainLooper(), callback)) + messenger?.send(msg) + } + } + + override fun classify(bitmap: Bitmap, rotation: Int): List { + val outputStream = ByteArrayOutputStream() + + bitmap.compress(Bitmap.CompressFormat.JPEG, 100, outputStream) + val info = TensorsInfo().apply { + addTensorInfo(NNStreamer.TensorType.UINT8, intArrayOf(outputStream.size(), 1)) + } + val data = TensorsData.allocate(info) + val byteBuffer = ByteBuffer.wrap(outputStream.toByteArray()) + val buffer = TensorsData.allocateByteBuffer(info.getTensorSize(0)) + + buffer.put(byteBuffer) + data.setTensorData(0, buffer) + MobileNetPipeline.pushInputData(data) + + return listOf() + } +} diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/domain/ObjectClassifier.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/domain/ObjectClassifier.kt new file mode 100644 index 0000000..f1913ca --- /dev/null +++ b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/domain/ObjectClassifier.kt @@ -0,0 +1,8 @@ +package ai.nnstreamer.ml.inference.offloading.domain + +import ai.nnstreamer.ml.inference.offloading.data.Classification +import android.graphics.Bitmap + +interface ObjectClassifier { + fun classify(bitmap: Bitmap, rotation: Int): List +} diff --git a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/ui/MainViewModel.kt b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/ui/MainViewModel.kt index 5b510cb..0d3546b 100644 --- a/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/ui/MainViewModel.kt +++ b/ml_inference_offloading/src/main/java/ai/nnstreamer/ml/inference/offloading/ui/MainViewModel.kt @@ -58,7 +58,7 @@ class MainViewModel @Inject constructor(private val offloadingServiceRepositoryI val newState = OffloadingServiceUiState(it.serviceId, it.state, it.port) newList.add(newState) } - _services.value = newList.toList() + _services.emit(newList) } } }