diff --git a/.github/workflows/tizen-web.yml b/.github/workflows/tizen-web.yml index ae0caccd..1a8b72a5 100644 --- a/.github/workflows/tizen-web.yml +++ b/.github/workflows/tizen-web.yml @@ -14,8 +14,11 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - app: ['ImageClassificationSingleShot', 'ImageClassificationPipeline', 'ImageClassificationOffloading'] + app: ['ImageClassificationSingleShot', 'ImageClassificationPipeline', 'ImageClassificationOffloading', 'ImageClassificationOffloadingYolo'] steps: + - uses: actions/setup-python@v5 + with: + python-version: '3.11' - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} @@ -25,6 +28,13 @@ jobs: wget -nc -O ${{ github.workspace }}/installer $TIZEN_STUDIO_URL chmod a+x ${{ github.workspace }}/installer bash ${{ github.workspace }}/installer --accept-license ${{ github.workspace }}/tizen-studio + - name: Download tflite model + if: ${{ matrix.app }} == 'ImageClassificationOffloadingYolo' + run: | + pip install ultralytics + python3 ${{ github.workspace }}/Tizen.web/${{ matrix.app }}/get_tflite_model.py + mv yolov8s_saved_model/yolov8s_float32.tflite ${{ github.workspace }}/Tizen.web/${{ matrix.app }}/res + shell: bash - name: Build Tizen web application shell: bash run: | diff --git a/Tizen.web/ImageClassificationOffloadingYolo/.project b/Tizen.web/ImageClassificationOffloadingYolo/.project new file mode 100644 index 00000000..c57a628e --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/.project @@ -0,0 +1,24 @@ + + + ImageClassificationOffloadingYolo + + + + + + json.validation.builder + + + + + org.tizen.web.project.builder.WebBuilder + + + + + + json.validation.nature + org.eclipse.wst.jsdt.core.jsNature + org.tizen.web.project.builder.WebNature + + diff --git a/Tizen.web/ImageClassificationOffloadingYolo/.tproject b/Tizen.web/ImageClassificationOffloadingYolo/.tproject new file mode 100644 index 00000000..0e708fec --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/.tproject @@ -0,0 +1,11 @@ + + + + + tizen-8.0 + + + + + + diff --git a/Tizen.web/ImageClassificationOffloadingYolo/README.md b/Tizen.web/ImageClassificationOffloadingYolo/README.md new file mode 100644 index 00000000..d2707c2d --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/README.md @@ -0,0 +1,18 @@ +# Image Classification Sample App (Offloading version) +## Description +* This is a sample application of Tizen ML Web APIs. +* If you want to run it on your device, Tizen 8.0 or higher is required. +* `appsrc` and `tensor_query_client` element are used. +* Tflite model is created by this [guide](https://github.com/nnstreamer/nnstreamer-example/tree/main/bash_script/example_yolo#export-to-tflite-and-torchscript-model-1). You should put exported model `yolov8s_float32.tflite` in res directory. +```py +from ultralytics import YOLO + +# Load a model +model = YOLO("yolov8s.pt") # load a pretrained model + +# Export the model +model.export(format="tflite", imgsz=224) # export the model to tflite format +``` + +## Demo +![Alt demo](./image_classification_offloading_yolo.png) diff --git a/Tizen.web/ImageClassificationOffloadingYolo/config.xml b/Tizen.web/ImageClassificationOffloadingYolo/config.xml new file mode 100644 index 00000000..c6e833fe --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/config.xml @@ -0,0 +1,13 @@ + + + + + + + ImageClassificationOffloadingYolo + + + + + + diff --git a/Tizen.web/ImageClassificationOffloadingYolo/css/style.css b/Tizen.web/ImageClassificationOffloadingYolo/css/style.css new file mode 100644 index 00000000..ffd019f7 --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/css/style.css @@ -0,0 +1,26 @@ +html, +body { + width: 100%; + height: 100%; + margin: 0 auto; + padding: 0; + background-color: #222222; + color: #ffffff; +} +#container { + display: flex; +} +.element { + flex: 1; + text-align: center; + font-size: 30px; +} +.button { + padding: 10px 10px; + font-size: 20px; + margin: 10px; + text-align: center; +} +.canvas { + margin: 50px +} diff --git a/Tizen.web/ImageClassificationOffloadingYolo/get_tflite_model.py b/Tizen.web/ImageClassificationOffloadingYolo/get_tflite_model.py new file mode 100644 index 00000000..a460b143 --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/get_tflite_model.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +""" +@file get_tflite_model.py +@date 24 Jun 2024 +@brief get yolov8 tflite model for Tizen Web application +@author Yelin Jeong +@bug No known bugs. + +""" + +from ultralytics import YOLO + +# Load a model +model = YOLO("yolov8s.pt") # load a pretrained model + +# Export the model +model.export(format="tflite", imgsz=224) # export the model to tflite format diff --git a/Tizen.web/ImageClassificationOffloadingYolo/icon.png b/Tizen.web/ImageClassificationOffloadingYolo/icon.png new file mode 100644 index 00000000..9765b1bd Binary files /dev/null and b/Tizen.web/ImageClassificationOffloadingYolo/icon.png differ diff --git a/Tizen.web/ImageClassificationOffloadingYolo/image_classification_offloading_yolo.png b/Tizen.web/ImageClassificationOffloadingYolo/image_classification_offloading_yolo.png new file mode 100644 index 00000000..cf4aa1ea Binary files /dev/null and b/Tizen.web/ImageClassificationOffloadingYolo/image_classification_offloading_yolo.png differ diff --git a/Tizen.web/ImageClassificationOffloadingYolo/images/tizen_32.png b/Tizen.web/ImageClassificationOffloadingYolo/images/tizen_32.png new file mode 100644 index 00000000..647c3f9f Binary files /dev/null and b/Tizen.web/ImageClassificationOffloadingYolo/images/tizen_32.png differ diff --git a/Tizen.web/ImageClassificationOffloadingYolo/index.html b/Tizen.web/ImageClassificationOffloadingYolo/index.html new file mode 100644 index 00000000..49e8bf89 --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/index.html @@ -0,0 +1,35 @@ + + + + + + + Tizen Image Classification Offloading Yolo Example + + + + + +

Image Classification YoloV8

+
+
+ +
+ +
Tab here
+
+
+
+ +
+ +
+ +
+ +
Tab here
+
+
+
+ + diff --git a/Tizen.web/ImageClassificationOffloadingYolo/js/main.js b/Tizen.web/ImageClassificationOffloadingYolo/js/main.js new file mode 100644 index 00000000..9dfa84ff --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/js/main.js @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +/** + * @file main.js + * @date 17 June 2024 + * @brief Yolo Image classification Offloading example + * @author Yelin Jeong + * @bug When drawing the results on the canvas, + * a blank image appears until the second attempts. + * (Only on Tizen RPI4 devices) + */ + +import { + getAbsPath, + getNetworkType, + getIpAddress, + GetImgPath, +} from "./utils.js"; + +let fHandle = null; +let tensorsData = null; +let tensorsInfo = null; + +function disposeData() { + if (fHandle != null) { + fHandle.close(); + } + + if (tensorsData != null) { + tensorsData.dispose(); + } + + if (tensorsInfo != null) { + tensorsInfo.dispose(); + } +} + +let localHandle; +let offloadingHandle; + +function createPipelineDescription(isLocal, filter) { + const absLabelPath = getAbsPath("coco.txt"); + + return ( + "appsrc caps=image/jpeg name=srcx_" + (isLocal ? "local" : "offloading") + " ! jpegdec ! " + + "videoconvert ! video/x-raw,format=RGB,width=224,height=224,framerate=0/1 ! tee name=t " + + "t. ! tensor_converter ! tensor_transform mode=arithmetic option=typecast:float32,div:255.0 ! " + + "other/tensors,num_tensors=1,format=static,dimensions=(string)3:224:224:1,types=float32,framerate=0/1 ! " + + filter + " ! " + "other/tensors,num_tensors=1,types=float32,format=static,dimensions=1029:84:1 ! " + + "tensor_transform mode=transpose option=1:0:2:3 ! " + + "other/tensors,num_tensors=1,types=float32,format=static,dimensions=84:1029:1 !" + + "tensor_decoder mode=bounding_boxes option1=yolov8 option2=" + absLabelPath + " option3=0 option4=224:224 option5=224:224 ! " + + "video/x-raw,width=224,height=224,format=RGBA,framerate=0/1 ! mix.sink_0 t. ! mix.sink_1 " + + "compositor name=mix sink_0::zorder=2 sink_1::zorder=1 ! video/x-raw,width=224,height=224,format=RGBA,framerate=0/1 ! tensor_converter ! " + + "other/tensors,num_tensors=1,types=uint8,format=static,dimensions=4:224:224:1 ! " + + "appsink sync=false name=sinkx_" + (isLocal ? "local" : "offloading") + ); +} + +/** + * Callback function for pipeline sink listener + */ +function sinkListenerCallback(sinkName, data) { + const endTime = performance.now(); + + const tensorsRetData = data.getTensorRawData(0).data; + const pixelData = new Uint8ClampedArray(tensorsRetData); + const imageData = new ImageData(pixelData, 224); + + let type; + if (sinkName.endsWith("local")) { + type = "local"; + } else { + type = "offloading"; + } + + const canvas = document.getElementById("canvas_" + type); + canvas.width = 224; + canvas.height = 224; + const ctx = canvas.getContext("2d"); + ctx.putImageData(imageData, 0, 0); + + const time = document.getElementById("time_" + type); + time.innerText = type + " : " + (endTime - startTime) + " ms"; +} + +/** + * Run a pipeline that uses Tizen device's resources + */ +function runLocal() { + const absModelPath = getAbsPath("yolov8s_float32.tflite"); + const filter = + "tensor_filter framework=tensorflow-lite model=" + absModelPath; + + const pipelineDescription = createPipelineDescription(true, filter); + + localHandle = tizen.ml.pipeline.createPipeline(pipelineDescription); + localHandle.start(); + localHandle.registerSinkListener("sinkx_local", sinkListenerCallback); +} + +/** + * Run a pipeline that uses other device's resources + */ +function runOffloading() { + const filter = + "tensor_query_client host=" + ip + + " port=" + document.getElementById("port").value + + " dest-host=" + document.getElementById("ip").value + + " dest-port=" + document.getElementById("port").value + + " timeout=1000"; + + const pipelineDescription = createPipelineDescription(false, filter); + + offloadingHandle = tizen.ml.pipeline.createPipeline(pipelineDescription); + offloadingHandle.start(); + offloadingHandle.registerSinkListener( + "sinkx_offloading", + sinkListenerCallback, + ); +} + +let startTime; + +/** + * Run a pipeline that uses other device's resources + */ +function inference(isLocal) { + const img_path = GetImgPath(); + let img = new Image(); + img.src = img_path; + + img.onload = function () { + disposeData(); + fHandle = tizen.filesystem.openFile("wgt-package" + img_path, "r"); + const imgUInt8Array = fHandle.readData(); + + tensorsInfo = new tizen.ml.TensorsInfo(); + tensorsInfo.addTensorInfo("tensor", "UINT8", [imgUInt8Array.length]); + tensorsData = tensorsInfo.getTensorsData(); + tensorsData.setTensorRawData(0, imgUInt8Array); + + startTime = performance.now(); + + if (isLocal) { + localHandle.getSource("srcx_local").inputData(tensorsData); + } else { + offloadingHandle.getSource("srcx_offloading").inputData(tensorsData); + } + }; +} + +let ip; + +window.onload = async function () { + const networkType = await getNetworkType(); + ip = await getIpAddress(networkType); + + document + .getElementById("start_local") + .addEventListener("click", function () { + runLocal(); + }); + + document + .getElementById("start_offloading") + .addEventListener("click", function () { + runOffloading(); + }); + + document + .getElementById("local") + .addEventListener("click", function () { + inference(true); + }); + + document + .getElementById("offloading") + .addEventListener("click", function () { + inference(false); + }); + + /* add eventListener for tizenhwkey */ + document.addEventListener("tizenhwkey", function (e) { + if (e.keyName === "back") { + try { + console.log("Pipeline is disposed!!"); + localHandle.stop(); + localHandle.dispose(); + + offloadingHandle.stop(); + offloadingHandle.dispose(); + + disposeData(); + + tizen.application.getCurrentApplication().exit(); + } catch (ignore) {} + } + }); +}; diff --git a/Tizen.web/ImageClassificationOffloadingYolo/js/utils.js b/Tizen.web/ImageClassificationOffloadingYolo/js/utils.js new file mode 100644 index 00000000..bdefef8d --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/js/utils.js @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +/** + * @file util.js + * @date 17 June 2024 + * @brief Utility function for Yolo Image Classification Offloading + * @author Yelin Jeong + */ + +/** + * Get absolute path for file + */ +export function getAbsPath(fileName) { + const filePath = 'wgt-package/res/' + fileName; + const URI_PREFIX = 'file://'; + const absPath = tizen.filesystem.toURI(filePath).substr(URI_PREFIX.length); + return absPath +} + +/** + * Get currently used network type + * @returns the network type + */ +export async function getNetworkType() { + return new Promise((resolve, reject) => { + tizen.systeminfo.getPropertyValue("NETWORK", function (data) { + resolve(data.networkType); + }); + }); +} + +/** + * Get IP address of the given network type + * @param networkType the network type used + * @returns ip address of the network type + */ +export async function getIpAddress(networkType) { + return new Promise((resolve, reject) => { + tizen.systeminfo.getPropertyValue( + networkType + "_NETWORK", + function (property) { + resolve(property.ipAddress); + }, + ); + }); +} + +/** + * Get the jpeg image path + * @returns image path + */ +export function GetImgPath() { + const MAX_IMG_CNT = 2; + let imgsrc = GetImgPath.count++ % MAX_IMG_CNT; + imgsrc = imgsrc.toString().concat('.jpg'); + return '/res/'.concat(imgsrc); +} +GetImgPath.count = 0; diff --git a/Tizen.web/ImageClassificationOffloadingYolo/res/0.jpg b/Tizen.web/ImageClassificationOffloadingYolo/res/0.jpg new file mode 100644 index 00000000..833d4a08 Binary files /dev/null and b/Tizen.web/ImageClassificationOffloadingYolo/res/0.jpg differ diff --git a/Tizen.web/ImageClassificationOffloadingYolo/res/1.jpg b/Tizen.web/ImageClassificationOffloadingYolo/res/1.jpg new file mode 100644 index 00000000..84b0c92a Binary files /dev/null and b/Tizen.web/ImageClassificationOffloadingYolo/res/1.jpg differ diff --git a/Tizen.web/ImageClassificationOffloadingYolo/res/coco.txt b/Tizen.web/ImageClassificationOffloadingYolo/res/coco.txt new file mode 100644 index 00000000..ec82f0ff --- /dev/null +++ b/Tizen.web/ImageClassificationOffloadingYolo/res/coco.txt @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +potted plant +bed +dining table +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush