Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Tizen/Web] Refactoring ImageClassificationOffloading example #339

Merged
merged 1 commit into from
Jun 12, 2024
Merged

[Tizen/Web] Refactoring ImageClassificationOffloading example #339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Tizen.web/ImageClassificationOffloading/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
<meta name="description" content="Tizen Image Classification SingleShot Example" />
<title>Tizen Image Classification Offloading Example</title>
<link rel="stylesheet" type="text/css" href="css/style.css" />
<script src="js/main.js"></script>
<script src="js/util.js" type="module"></script>
<script src="js/main.js" type="module"></script>
</head>
<body>
<h1 align="center" margin-bottom="50px">Image Classification</h1>
Expand Down
278 changes: 114 additions & 164 deletions Tizen.web/ImageClassificationOffloading/js/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* SPDX-License-Identifier: Apache-2.0-only */
/* SPDX-License-Identifier: Apache-2.0 */

/**
* @file main.js
Expand All @@ -7,56 +7,67 @@
* @author Yelin Jeong <[email protected]>
*/

var localSrc;
var remoteSrc;
var labels;
var startTime;
var ctx;
var label;
import {
getNetworkType,
getIpAddress,
GetMaxIdx,
GetImgPath,
loadLabelInfo,
} from './utils.js';

/**
* Find the index of maximum value in the given array
* @param array the score list of each class
* @returns the index of the maximum value
*/
function GetMaxIdx(array) {
if (array.length === 0) {
console.log('array length zero')
return -1;
let fHandle = null;
let tensorsData = null;
let tensorsInfo = null;

function disposeData() {
if (fHandle != null) {
fHandle.close();
}

var max = array[0];
var maxIdx = 0;
if (tensorsData != null) {
tensorsData.dispose();
}

for (var i = 0; i < array.length; ++i) {
if (array[i] > max) {
maxIdx = i;
max = array[i];
}
if (tensorsInfo != null) {
tensorsInfo.dispose();
}
return maxIdx;
}

/**
* Get the jpeg image path
* @returns image path
*/
function GetImgPath() {
const MAX_IMG_CNT = 2;
var imgsrc = GetImgPath.count++ % MAX_IMG_CNT;
imgsrc = imgsrc.toString().concat('.jpg');
return '/res/'.concat(imgsrc);
let localHandle;
let offloadingHandle;

function createPipelineDescription(isLocal, filter) {
return (
'appsrc caps=image/jpeg name=srcx_' + (isLocal ? 'local' : 'offloading') + ' ! jpegdec ! ' +
'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' +
'other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1 ! ' + filter + ' ! ' +
'other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! ' +
'tensor_sink name=sinkx_' + (isLocal ? 'local' : 'offloading')
);
}
GetImgPath.count = 0;

/**
* Load the label from the text file and return the string array
* @returns string array
* Callback function for pipeline sink listener
*/
function loadLabelInfo() {
var fHandle = tizen.filesystem.openFile('wgt-package/res/labels.txt', 'r');
var labelList = fHandle.readString();
return labelList.split('\n');
function sinkListenerCallback(sinkName, data) {
const endTime = performance.now();

const tensorsRetData = data.getTensorRawData(0);
const maxIdx = GetMaxIdx(tensorsRetData.data);

let type;
if (sinkName.endsWith('local')) {
type = 'local';
}
else {
type = 'offloading';
}

const label = document.getElementById('label_' + type);
label.innerText = labels[maxIdx];

const time = document.getElementById('time_' + type);
time.innerText = type + ' : ' + (endTime - startTime) + ' ms';
}

/**
Expand All @@ -66,109 +77,37 @@ function runLocal() {
const modelPath = 'wgt-package/res/mobilenet_v1_1.0_224_quant.tflite';
const URI_PREFIX = 'file://';
const absModelPath = tizen.filesystem.toURI(modelPath).substr(URI_PREFIX.length);
const filter = 'tensor_filter framework=tensorflow-lite model=' + absModelPath;

const pipelineDescription = 'appsrc caps=image/jpeg name=srcx_local ! jpegdec ! ' +
'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' +
'tensor_filter framework=tensorflow-lite model=' + absModelPath + ' ! ' +
'appsink name=sinkx_local';

const pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
pHandle.start();

localSrc = pHandle.getSource('srcx_local');

pHandle.registerSinkListener('sinkx_local', function(sinkName, data) {
const endTime = performance.now();
const label = document.getElementById('label_local')
const tensorsRetData = data.getTensorRawData(0);
const maxIdx = GetMaxIdx(tensorsRetData.data);
label.innerText = labels[maxIdx];

const time = document.getElementById('time_local');
time.innerText = 'local : ' + (endTime - startTime) + ' ms'
});
}
const pipelineDescription = createPipelineDescription(true,filter);

let ip;
async function getNetworkType() {
return new Promise((resolve, reject) => {
tizen.systeminfo.getPropertyValue("NETWORK", function (data) {
resolve(data.networkType);
});
});
}

async function getIpAddress(networkType) {
return new Promise((resolve, reject) => {
tizen.systeminfo.getPropertyValue(
networkType + "_NETWORK",
function (property) {
resolve(property.ipAddress);
},
);
});
}

async function setIpAddress() {
try {
const networkType = await getNetworkType();
ip = await getIpAddress(networkType);
console.log(ip);
}
catch (e) {
console.error("Error getting IP address:", error);
}
localHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
localHandle.start();
localHandle.registerSinkListener('sinkx_local', sinkListenerCallback);
}

/**
* Run a pipeline that uses other device's resources
*/
async function runRemote() {
await setIpAddress();

const pipelineDescription = 'appsrc caps=image/jpeg name=srcx_remote ! jpegdec ! ' +
'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' +
'other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1 ! ' +
'tensor_query_client host='+ ip +' port=' + document.getElementById('port').value + ' dest-host=' + document.getElementById('ip').value + ' ' +
'dest-port=' + document.getElementById('port').value + ' timeout=1000 ! ' +
'other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! tensor_sink name=sinkx_remote';

const pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
pHandle.start();

remoteSrc = pHandle.getSource('srcx_remote');

pHandle.registerSinkListener('sinkx_remote', function(sinkName, data) {
const endTime = performance.now();
const label = document.getElementById('label_offloading');
const tensorsRetData = data.getTensorRawData(0);
const maxIdx = GetMaxIdx(tensorsRetData.data);
label.innerText = labels[maxIdx];

const time = document.getElementById('time_offloading');
time.innerText = 'offloading : ' + (endTime - startTime) + ' ms'
});
}
function runOffloading() {
const filter = 'tensor_query_client host=' + ip + ' port=' + document.getElementById('port').value +
' dest-host=' + document.getElementById('ip').value +
' dest-port=' + document.getElementById('port').value +
' timeout=1000';

let fHandle = null;
let tensorsData = null;
let tensorsInfo = null;
const pipelineDescription = createPipelineDescription(false,filter);

function disposeData() {
if (fHandle != null) {
fHandle.close();
}

if (tensorsData != null) {
tensorsData.dispose();
}

if (tensorsInfo != null) {
tensorsInfo.dispose();
}
offloadingHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
offloadingHandle.start();
offloadingHandle.registerSinkListener('sinkx_offloading', sinkListenerCallback);
}

function inference(src, canvas) {
let startTime;

/**
* Run a pipeline that uses other device's resources
*/
function inference(isLocal) {
const img_path = GetImgPath();
let img = new Image();
img.src = img_path;
Expand All @@ -183,52 +122,63 @@ function inference(src, canvas) {
tensorsData = tensorsInfo.getTensorsData();
tensorsData.setTensorRawData(0, imgUInt8Array);

startTime = performance.now()
src.inputData(tensorsData);
startTime = performance.now();

if (isLocal) {
localHandle.getSource('srcx_local').inputData(tensorsData);
} else {
offloadingHandle.getSource('srcx_offloading').inputData(tensorsData);
}

const canvasName = 'canvas_' + (isLocal ? 'local' : 'offloading');
const canvas = document.getElementById(canvasName);
const ctx = canvas.getContext('2d');
ctx.drawImage(img, 0, 0);
}
ctx.drawImage(img, 0, 0);
};
}

window.onload = function() {
labels = loadLabelInfo();

const btnLocal = document.getElementById('start_local');

btnLocal.addEventListener('click', function() {
runLocal();
});

const btnOffloading = document.getElementById('start_offloading');
let ip;
let labels;

btnOffloading.addEventListener('click', function() {
runRemote();
});
window.onload = async function () {
const networkType = await getNetworkType();
ip = await getIpAddress(networkType);
labels = loadLabelInfo();

const localPage = document.getElementById('local');
document
.getElementById('start_local')
.addEventListener('click', function () {
runLocal();
});

localPage.addEventListener('click', function() {
if (localSrc) {
inference(localSrc, document.getElementById('canvas_local'));
}
});
document
.getElementById('start_offloading')
.addEventListener('click', function () {
runOffloading();
});

const offloadingPage = document.getElementById('offloading');
document
.getElementById('local')
.addEventListener('click', function () {
inference(true);
});

offloadingPage.addEventListener('click', function() {
if (remoteSrc) {
inference(remoteSrc, document.getElementById('canvas_offloading'));
}
});
document
.getElementById('offloading')
.addEventListener('click', function () {
inference(false);
});

/* add eventListener for tizenhwkey */
document.addEventListener('tizenhwkey', function(e) {
document.addEventListener('tizenhwkey', function (e) {
if (e.keyName === 'back') {
try {
console.log('Pipeline is disposed!!');
pHandle.stop();
pHandle.dispose();
localHandle.stop();
localHandle.dispose();

offloadingHandle.stop();
offloadingHandle.dispose();

disposeData();

Expand Down
Loading