From 85f3793d37c3dd9df4e80b43a87e06467b2d03b1 Mon Sep 17 00:00:00 2001 From: Yelin Jeong Date: Fri, 7 Jun 2024 17:12:29 +0900 Subject: [PATCH] [Tizen/Web] Refactoring ImageClassificationOffloading example - Create utility module - Functionalization of duplicated part - Remove unused variables - Change var variables to use let type - Declare variables closer to where they are used Signed-off-by: Yelin Jeong --- .../ImageClassificationOffloading/index.html | 3 +- .../ImageClassificationOffloading/js/main.js | 276 +++++++----------- .../ImageClassificationOffloading/js/utils.js | 81 +++++ 3 files changed, 196 insertions(+), 164 deletions(-) create mode 100644 Tizen.web/ImageClassificationOffloading/js/utils.js diff --git a/Tizen.web/ImageClassificationOffloading/index.html b/Tizen.web/ImageClassificationOffloading/index.html index 5703de7b..375495ef 100644 --- a/Tizen.web/ImageClassificationOffloading/index.html +++ b/Tizen.web/ImageClassificationOffloading/index.html @@ -6,7 +6,8 @@ Tizen Image Classification Offloading Example - + +

Image Classification

diff --git a/Tizen.web/ImageClassificationOffloading/js/main.js b/Tizen.web/ImageClassificationOffloading/js/main.js index 8a936243..459bd665 100644 --- a/Tizen.web/ImageClassificationOffloading/js/main.js +++ b/Tizen.web/ImageClassificationOffloading/js/main.js @@ -7,56 +7,67 @@ * @author Yelin Jeong */ -var localSrc; -var remoteSrc; -var labels; -var startTime; -var ctx; -var label; +import { + getNetworkType, + getIpAddress, + GetMaxIdx, + GetImgPath, + loadLabelInfo, +} from './utils.js'; -/** - * Find the index of maximum value in the given array - * @param array the score list of each class - * @returns the index of the maximum value - */ -function GetMaxIdx(array) { - if (array.length === 0) { - console.log('array length zero') - return -1; +let fHandle = null; +let tensorsData = null; +let tensorsInfo = null; + +function disposeData() { + if (fHandle != null) { + fHandle.close(); } - var max = array[0]; - var maxIdx = 0; + if (tensorsData != null) { + tensorsData.dispose(); + } - for (var i = 0; i < array.length; ++i) { - if (array[i] > max) { - maxIdx = i; - max = array[i]; - } + if (tensorsInfo != null) { + tensorsInfo.dispose(); } - return maxIdx; } -/** - * Get the jpeg image path - * @returns image path - */ -function GetImgPath() { - const MAX_IMG_CNT = 2; - var imgsrc = GetImgPath.count++ % MAX_IMG_CNT; - imgsrc = imgsrc.toString().concat('.jpg'); - return '/res/'.concat(imgsrc); +let localHandle; +let offloadingHandle; + +function createPipelineDescription(isLocal, filter) { + return ( + 'appsrc caps=image/jpeg name=srcx_' + (isLocal ? 'local' : 'offloading') + ' ! jpegdec ! ' + + 'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' + + 'other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1 ! ' + filter + ' ! ' + + 'other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! ' + + 'tensor_sink name=sinkx_' + (isLocal ? 'local' : 'offloading') + ); } -GetImgPath.count = 0; /** - * Load the label from the text file and return the string array - * @returns string array + * Callback function for pipeline sink listener */ -function loadLabelInfo() { - var fHandle = tizen.filesystem.openFile('wgt-package/res/labels.txt', 'r'); - var labelList = fHandle.readString(); - return labelList.split('\n'); +function sinkListenerCallback(sinkName, data) { + const endTime = performance.now(); + + const tensorsRetData = data.getTensorRawData(0); + const maxIdx = GetMaxIdx(tensorsRetData.data); + + let type; + if (sinkName.endsWith('local')) { + type = 'local'; + } + else { + type = 'offloading'; + } + + const label = document.getElementById('label_' + type); + label.innerText = labels[maxIdx]; + + const time = document.getElementById('time_' + type); + time.innerText = type + ' : ' + (endTime - startTime) + ' ms'; } /** @@ -66,109 +77,37 @@ function runLocal() { const modelPath = 'wgt-package/res/mobilenet_v1_1.0_224_quant.tflite'; const URI_PREFIX = 'file://'; const absModelPath = tizen.filesystem.toURI(modelPath).substr(URI_PREFIX.length); + const filter = 'tensor_filter framework=tensorflow-lite model=' + absModelPath; - const pipelineDescription = 'appsrc caps=image/jpeg name=srcx_local ! jpegdec ! ' + - 'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' + - 'tensor_filter framework=tensorflow-lite model=' + absModelPath + ' ! ' + - 'appsink name=sinkx_local'; - - const pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription); - pHandle.start(); - - localSrc = pHandle.getSource('srcx_local'); - - pHandle.registerSinkListener('sinkx_local', function(sinkName, data) { - const endTime = performance.now(); - const label = document.getElementById('label_local') - const tensorsRetData = data.getTensorRawData(0); - const maxIdx = GetMaxIdx(tensorsRetData.data); - label.innerText = labels[maxIdx]; - - const time = document.getElementById('time_local'); - time.innerText = 'local : ' + (endTime - startTime) + ' ms' - }); -} + const pipelineDescription = createPipelineDescription(true,filter); -let ip; -async function getNetworkType() { - return new Promise((resolve, reject) => { - tizen.systeminfo.getPropertyValue("NETWORK", function (data) { - resolve(data.networkType); - }); - }); -} - -async function getIpAddress(networkType) { - return new Promise((resolve, reject) => { - tizen.systeminfo.getPropertyValue( - networkType + "_NETWORK", - function (property) { - resolve(property.ipAddress); - }, - ); - }); -} - -async function setIpAddress() { - try { - const networkType = await getNetworkType(); - ip = await getIpAddress(networkType); - console.log(ip); - } - catch (e) { - console.error("Error getting IP address:", error); - } + localHandle = tizen.ml.pipeline.createPipeline(pipelineDescription); + localHandle.start(); + localHandle.registerSinkListener('sinkx_local', sinkListenerCallback); } /** * Run a pipeline that uses other device's resources */ -async function runRemote() { - await setIpAddress(); - - const pipelineDescription = 'appsrc caps=image/jpeg name=srcx_remote ! jpegdec ! ' + - 'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' + - 'other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1 ! ' + - 'tensor_query_client host='+ ip +' port=' + document.getElementById('port').value + ' dest-host=' + document.getElementById('ip').value + ' ' + - 'dest-port=' + document.getElementById('port').value + ' timeout=1000 ! ' + - 'other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! tensor_sink name=sinkx_remote'; - - const pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription); - pHandle.start(); - - remoteSrc = pHandle.getSource('srcx_remote'); - - pHandle.registerSinkListener('sinkx_remote', function(sinkName, data) { - const endTime = performance.now(); - const label = document.getElementById('label_offloading'); - const tensorsRetData = data.getTensorRawData(0); - const maxIdx = GetMaxIdx(tensorsRetData.data); - label.innerText = labels[maxIdx]; - - const time = document.getElementById('time_offloading'); - time.innerText = 'offloading : ' + (endTime - startTime) + ' ms' - }); -} +async function runOffloading() { + const filter = 'tensor_query_client host=' + ip + ' port=' + document.getElementById('port').value + + ' dest-host=' + document.getElementById('ip').value + + ' dest-port=' + document.getElementById('port').value + + ' timeout=1000'; -let fHandle = null; -let tensorsData = null; -let tensorsInfo = null; + const pipelineDescription = createPipelineDescription(false,filter); -function disposeData() { - if (fHandle != null) { - fHandle.close(); - } - - if (tensorsData != null) { - tensorsData.dispose(); - } - - if (tensorsInfo != null) { - tensorsInfo.dispose(); - } + offloadingHandle = tizen.ml.pipeline.createPipeline(pipelineDescription); + offloadingHandle.start(); + offloadingHandle.registerSinkListener('sinkx_offloading', sinkListenerCallback); } -function inference(src, canvas) { +let startTime; + +/** + * Run a pipeline that uses other device's resources + */ +function inference(isLocal) { const img_path = GetImgPath(); let img = new Image(); img.src = img_path; @@ -183,52 +122,63 @@ function inference(src, canvas) { tensorsData = tensorsInfo.getTensorsData(); tensorsData.setTensorRawData(0, imgUInt8Array); - startTime = performance.now() - src.inputData(tensorsData); + startTime = performance.now(); + if (isLocal) { + localHandle.getSource('srcx_local').inputData(tensorsData); + } else { + offloadingHandle.getSource('srcx_offloading').inputData(tensorsData); + } + + const canvasName = 'canvas_' + (isLocal ? 'local' : 'offloading'); + const canvas = document.getElementById(canvasName); const ctx = canvas.getContext('2d'); - ctx.drawImage(img, 0, 0); - } + ctx.drawImage(img, 0, 0); + }; } -window.onload = function() { - labels = loadLabelInfo(); - - const btnLocal = document.getElementById('start_local'); - - btnLocal.addEventListener('click', function() { - runLocal(); - }); - - const btnOffloading = document.getElementById('start_offloading'); +let ip; +let labels; - btnOffloading.addEventListener('click', function() { - runRemote(); - }); +window.onload = async function () { + const networkType = await getNetworkType(); + ip = await getIpAddress(networkType); + labels = loadLabelInfo(); - const localPage = document.getElementById('local'); + document + .getElementById('start_local') + .addEventListener('click', function () { + runLocal(); + }); - localPage.addEventListener('click', function() { - if (localSrc) { - inference(localSrc, document.getElementById('canvas_local')); - } - }); + document + .getElementById('start_offloading') + .addEventListener('click', function () { + runOffloading(); + }); - const offloadingPage = document.getElementById('offloading'); + document + .getElementById('local') + .addEventListener('click', function () { + inference(true); + }); - offloadingPage.addEventListener('click', function() { - if (remoteSrc) { - inference(remoteSrc, document.getElementById('canvas_offloading')); - } - }); + document + .getElementById('offloading') + .addEventListener('click', function () { + inference(false); + }); /* add eventListener for tizenhwkey */ - document.addEventListener('tizenhwkey', function(e) { + document.addEventListener('tizenhwkey', function (e) { if (e.keyName === 'back') { try { console.log('Pipeline is disposed!!'); - pHandle.stop(); - pHandle.dispose(); + localHandle.stop(); + localHandle.dispose(); + + offloadingHandle.stop(); + offloadingHandle.dispose(); disposeData(); diff --git a/Tizen.web/ImageClassificationOffloading/js/utils.js b/Tizen.web/ImageClassificationOffloading/js/utils.js new file mode 100644 index 00000000..26d338ab --- /dev/null +++ b/Tizen.web/ImageClassificationOffloading/js/utils.js @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: Apache-2.0-only */ + +/** + * @file util.js + * @date 7 June 2024 + * @brief Utility function for Image Classification Offloading + * @author Yelin Jeong + */ + +/** + * Get currently used network type + * @returns the network type + */ +export async function getNetworkType() { + return new Promise((resolve, reject) => { + tizen.systeminfo.getPropertyValue("NETWORK", function (data) { + resolve(data.networkType); + }); + }); +} + +/** + * Get IP address of the given network type + * @param networkType the network type used + * @returns ip address of the network type + */ +export async function getIpAddress(networkType) { + return new Promise((resolve, reject) => { + tizen.systeminfo.getPropertyValue( + networkType + "_NETWORK", + function (property) { + resolve(property.ipAddress); + }, + ); + }); +} + +/** + * Find the index of maximum value in the given array + * @param array the score list of each class + * @returns the index of the maximum value + */ +export function GetMaxIdx(array) { + if (array.length === 0) { + console.log('array length zero') + return -1; + } + + var max = array[0]; + var maxIdx = 0; + + for (var i = 0; i < array.length; ++i) { + if (array[i] > max) { + maxIdx = i; + max = array[i]; + } + } + return maxIdx; +} + +/** + * Get the jpeg image path + * @returns image path + */ +export function GetImgPath() { + const MAX_IMG_CNT = 2; + var imgsrc = GetImgPath.count++ % MAX_IMG_CNT; + imgsrc = imgsrc.toString().concat('.jpg'); + return '/res/'.concat(imgsrc); +} +GetImgPath.count = 0; + +/** + * Load the label from the text file and return the string array + * @returns string array + */ +export function loadLabelInfo() { + var fHandle = tizen.filesystem.openFile('wgt-package/res/labels.txt', 'r'); + var labelList = fHandle.readString(); + return labelList.split('\n'); +}