Testing OpenCV TrackerMIL 'Tracking error' on high resolution (#8934)

cvat-ai · Feb 12, 2025 · 26446ae · 26446ae
1 parent df5678a
commit 26446ae
Show file tree

Hide file tree

Showing 9 changed files with 92 additions and 31 deletions.
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -30,7 +30,7 @@
                 "background": {
                   "activeOnStart": true,
                   "beginsPattern": "webpack-dev-server",
-                  "endsPattern": "Compiled"
+                  "endsPattern": "compiled"
                 }
             }
         }

diff --git a/cvat-ui/src/utils/opencv-wrapper/tracker-mil.ts b/cvat-ui/src/utils/opencv-wrapper/tracker-mil.ts
@@ -20,7 +20,7 @@ export default class TrackerMILImplementation implements TrackerMIL {
         this.cv = cv;
         this.trackerMIL = new cv.TrackerMIL();
         this.imageData = null;
-        this.maxSize = 2560;
+        this.maxSize = 1080;
         this.imageScale = 1;
         this.name = 'TrackerMIL';
     }

diff --git a/tests/cypress/e2e/actions_tasks2/case_101_opencv_basic_actions.js b/tests/cypress/e2e/actions_tasks2/case_101_opencv_basic_actions.js
@@ -57,23 +57,33 @@ context('OpenCV. Intelligent scissors. Histogram Equalization. TrackerMIL.', ()
     const textDefaultValue = 'Some default value for type Text';
     const imagesCount = 5;
     const imageFileName = `image_${labelName.replace(' ', '_').toLowerCase()}`;
-    const width = 400;
-    const height = 400;
-    const posX = 10;
-    const posY = 10;
-    const color = 'gray';
+    const width = 5000;
+    const height = 5000;
+    const delta = 5;
+    const step = 62.5;
+    const maxTextWidth = undefined;
+    const textHeightPx = 981.25;
+    const posX = 125;
+    const posY = 125;
     const archiveName = `${imageFileName}.zip`;
     const archivePath = `cypress/fixtures/${archiveName}`;
     const imagesFolder = `cypress/fixtures/${imageFileName}`;
     const directoryToArchive = imagesFolder;
     const extension = 'jpg';
+    const color = 'grey';
+    const textColor = 'black';
+    const fontSize = textHeightPx;
 
     before(() => {
         cy.visit('/auth/login');
         cy.login();
         for (let i = 0; i < imagesCount; i++) {
-            cy.imageGenerator(imagesFolder, imageFileName + i, width, height, color, posX + i * 5,
-                posY + i * 5, labelName, 1, extension);
+            cy.makeCustomImage(imagesFolder, `${imageFileName}_${i}`,
+                width, height,
+                fontSize, color, textColor,
+                posX + i * step, posY + i * step,
+                `${labelName}. Num ${i}`, extension,
+                maxTextWidth);
         }
         cy.createZipArchive(directoryToArchive, archivePath);
         cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, archiveName);
@@ -114,7 +124,7 @@ context('OpenCV. Intelligent scissors. Histogram Equalization. TrackerMIL.', ()
                     .find('[role="slider"]')
                     .type(generateString(4, 'rightarrow'));
                 cy.get('.cvat_canvas_interact_intermediate_shape').then((_intermediateShape) => {
-                    // Get count of points againe
+                    // Get count of points again
                     const intermediateShapeNumberPointsAfterChange = _intermediateShape.attr('points').split(' ').length;
                     // expected 7 to be below 10
                     expect(intermediateShapeNumberPointsBeforeChange).to.be.lt(
@@ -212,7 +222,7 @@ context('OpenCV. Intelligent scissors. Histogram Equalization. TrackerMIL.', ()
 
         it('Create a shape with "TrackerMIL". Track it for several frames.', () => {
             cy.createRectangle(createRectangleTrack2Points);
-            // We will start testing tracking from 2-d frame because it's a bit unstable on inintialization
+            // We will start testing tracking from 2nd frame because it's a bit unstable on inintialization
             cy.useOpenCVTracker({ tracker: 'TrackerMIL', targetFrame: 4 });
             cy.get('#cvat_canvas_shape_4')
                 .then((shape) => {
@@ -221,13 +231,11 @@ context('OpenCV. Intelligent scissors. Histogram Equalization. TrackerMIL.', ()
                     for (let i = 1; i < imagesCount; i++) {
                         cy.goToNextFrame(i);
                         // In the beginning of this test we created images with text
-                        // On each frame text is moved by 5px on x and y axis,
+                        // On each frame text is moved by `${step}px` on x and y axis,
                         // so we expect shape to be close to real text positions
-                        cy.get('#cvat_canvas_shape_4').invoke('attr', 'x').then((xVal) => {
-                            expect(parseFloat(xVal)).to.be.closeTo(x + i * 5, 3.0);
-                        });
-                        cy.get('#cvat_canvas_shape_4').invoke('attr', 'y').then((yVal) => {
-                            expect(parseFloat(yVal)).to.be.closeTo(y + i * 5, 3.0);
+                        cy.get('#cvat_canvas_shape_4').then(($shape) => {
+                            expect(+$shape.attr('x')).to.be.closeTo(x + i * step, delta);
+                            expect(+$shape.attr('y')).to.be.closeTo(y + i * step, delta);
                         });
                         cy.get('#cvat-objects-sidebar-state-item-4')
                             .should('contain', 'RECTANGLE TRACK')

diff --git a/tests/cypress/plugins/imageGenerator/addPlugin.js b/tests/cypress/plugins/imageGenerator/addPlugin.js
@@ -2,10 +2,12 @@
 //
 // SPDX-License-Identifier: MIT
 
-// eslint-disable-next-line no-use-before-define
+/* eslint no-use-before-define: 0 */
 exports.imageGenerator = imageGenerator;
+exports.bufferToImage = bufferToImage;
 
 const path = require('path');
+const fs = require('fs-extra');
 const jimp = require('jimp');
 
 function createImage(width, height, color) {
@@ -17,6 +19,15 @@ function createImage(width, height, color) {
         }));
     });
 }
+function createImageFromBuffer(bitmapObj) {
+    return new Promise((resolve, reject) => {
+        // eslint-disable-next-line new-cap, no-new
+        new jimp(bitmapObj, (err, image) => {
+            if (err) reject(err);
+            resolve(image);
+        });
+    });
+}
 
 function appendText(image, posX, posY, message, index) {
     return new Promise((resolve, reject) => {
@@ -33,13 +44,22 @@ async function imageGenerator(args) {
         directory, fileName, width, height, color, posX, posY, message, count, extension,
     } = args;
     const file = path.join(directory, fileName);
-    try {
-        for (let i = 1; i <= count; i++) {
-            let image = await createImage(width, height, color);
-            image = await appendText(image, posX, posY, message, i);
-            image.write(`${file}_${i}.${extension}`);
-        }
-    // eslint-disable-next-line no-empty
-    } catch (e) {}
+    for (let i = 1; i <= count; i++) {
+        let image = await createImage(width, height, color);
+        image = await appendText(image, posX, posY, message, i);
+        image.write(`${file}_${i}.${extension}`);
+    }
     return null;
 }
+
+async function bufferToImage(args) {
+    const {
+        directory, fileName, extension, buffer,
+    } = args;
+    let file = null;
+    fs.mkdirp(directory);
+    file = path.join(directory, `${fileName}.${extension}`);
+    const image = await createImageFromBuffer(Buffer.from(buffer.data));
+    image.write(file);
+    return fs.pathExists(file);
+}
diff --git a/tests/cypress/plugins/imageGenerator/imageGeneratorCommand.js b/tests/cypress/plugins/imageGenerator/imageGeneratorCommand.js
@@ -14,3 +14,10 @@ Cypress.Commands.add('imageGenerator', (directory, fileName, width, height, colo
     count,
     extension,
 }));
+
+Cypress.Commands.add('bufferToImage', (directory, fileName, extension = 'png', buffer = null) => cy.task('bufferToImage', {
+    directory,
+    fileName,
+    extension,
+    buffer,
+}));
diff --git a/tests/cypress/plugins/index.js b/tests/cypress/plugins/index.js
@@ -7,7 +7,7 @@
 const fs = require('fs');
 // eslint-disable-next-line import/no-extraneous-dependencies
 const { isFileExist } = require('cy-verify-downloads');
-const { imageGenerator } = require('./imageGenerator/addPlugin');
+const { imageGenerator, bufferToImage } = require('./imageGenerator/addPlugin');
 const { createZipArchive } = require('./createZipArchive/addPlugin');
 const { compareImages } = require('./compareImages/addPlugin');
 const { unpackZipArchive } = require('./unpackZipArchive/addPlugin');
@@ -19,6 +19,7 @@ module.exports = (on, config) => {
     on('task', { createZipArchive });
     on('task', { compareImages });
     on('task', { unpackZipArchive });
+    on('task', { bufferToImage });
     on('task', {
         log(message) {
             console.log(message);
@@ -27,6 +28,7 @@ module.exports = (on, config) => {
     });
     on('task', {
         listFiles(folderName) {
+            // eslint-disable-next-line security/detect-non-literal-fs-filename
             return fs.readdirSync(folderName);
         },
     });
@@ -43,6 +45,7 @@ module.exports = (on, config) => {
     });
     on('after:spec', (spec, results) => {
         if (results && results.stats.failures === 0 && results.video) {
+            // eslint-disable-next-line security/detect-non-literal-fs-filename
             fs.unlinkSync(results.video);
         }
     });

diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js
@@ -1789,3 +1789,25 @@ Cypress.Commands.add('clickSaveAnnotationView', () => {
     cy.get('button').contains('Save').click();
     cy.get('button').contains('Save').trigger('mouseout');
 });
+
+Cypress.Commands.add('makeCustomImage', (directory, fileName,
+    width, height, fontSize, backColor, textColor, posX, posY,
+    message, extension = 'png', maxWidth = undefined) => {
+    cy.window().then(async ($win) => {
+        const ofc = new $win.OffscreenCanvas(width, height);
+        const ctx = ofc.getContext('2d');
+
+        ctx.fillStyle = backColor;
+        ctx.fillRect(0, 0, width, height);
+
+        ctx.font = `${fontSize}px Impact`;
+
+        ctx.fillStyle = textColor;
+        ctx.textBaseline = 'top'; // so that text aligns with tracking coords
+        ctx.fillText(message, posX, posY, maxWidth);
+
+        cy.bufferToImage(directory, fileName, extension,
+            await (await ofc.convertToBlob()).arrayBuffer(),
+        );
+    });
+});
diff --git a/tests/package.json b/tests/package.json
@@ -16,5 +16,6 @@
     "fs-extra": "^10.1.0",
     "jimp": "^0.22.10"
   },
-  "version": "0.0.0"
+  "version": "0.0.0",
+  "devDependencies": {}
 }
diff --git a/yarn.lock b/yarn.lock
@@ -3230,9 +3230,9 @@ camera-controls@^1.25.3:
   integrity sha512-EfzbovxLssyWpJVG9uKcazSDDIEcd1hUsPhPF/OWWnICsKY9WbLY/2S4UPW73HHbvnVeR/Z9wsWaQKtANy/2Yg==
 
 caniuse-lite@^1.0.30001587, caniuse-lite@^1.0.30001599:
-  version "1.0.30001617"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001617.tgz#809bc25f3f5027ceb33142a7d6c40759d7a901eb"
-  integrity sha512-mLyjzNI9I+Pix8zwcrpxEbGlfqOkF9kM3ptzmKNw5tizSyYwMe+nGLTqMK9cO+0E+Bh6TsBxNAaHWEM8xwSsmA==
+  version "1.0.30001690"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001690.tgz"
+  integrity sha512-5ExiE3qQN6oF8Clf8ifIDcMRCRE/dMGcETG/XGMD8/XiXm6HXQgQTh1yZYLXXpSOsEUlJm1Xr7kGULZTuGtP/w==
 
 canvas@^2.8.0:
   version "2.11.2"