added drawCorners and related code

- Not working properly
webarkit · Oct 21, 2024 · 3291aa2 · 3291aa2
1 parent 17e64ce
commit 3291aa2
Show file tree

Hide file tree

Showing 9 changed files with 76 additions and 25 deletions.
diff --git a/dist/src/Workers/WebARKitCVWorkers.js b/dist/src/Workers/WebARKitCVWorkers.js
diff --git a/dist/src/Workers/WebARKitCVWorkers.js.map b/dist/src/Workers/WebARKitCVWorkers.js.map
diff --git a/dist/src/core/WebARKitCoreCV.js b/dist/src/core/WebARKitCoreCV.js
diff --git a/dist/src/core/WebARKitCoreCV.js.map b/dist/src/core/WebARKitCoreCV.js.map
diff --git a/dist/webarkitCV.js b/dist/webarkitCV.js
diff --git a/examples/example_image.html b/examples/example_image.html
@@ -17,6 +17,10 @@
 
         const webarkit = new WebARKitCV.WebARKitCV();
         console.log(webarkit);
+        var overlayCanvas;
+        var oWidth = 375;
+        var oHeight = 375;
+        createOverlayCanvas();
 
         webarkit.setWidth(375)
             .setHeight(450)
@@ -33,10 +37,49 @@
                 window.addEventListener("getMatrix", function (e) {
                     console.log(e.detail);
                     transformElem(e.detail.matrix, imgElem);
+                    drawCorners(e.detail.corners);
                 }
                 )
             })
         }
+        function setVideoStyle(elem) {
+            elem.style.position = "absolute";
+            elem.style.top = 0;
+            elem.style.left = 0;
+        }
+        function createOverlayCanvas() {
+            overlayCanvas = document.createElement("canvas");
+            setVideoStyle(overlayCanvas);
+            overlayCanvas.id = "overlay";
+            overlayCanvas.width = oWidth;
+            overlayCanvas.height = oHeight;
+            overlayCanvas.style.zIndex = 1;
+            document.body.appendChild(overlayCanvas);
+        }
+
+        function clearOverlayCtx() {
+            const overlayCtx = overlayCanvas.getContext("2d");
+            overlayCtx.clearRect(0, 0, oWidth, oHeight);
+        }
+
+        function drawCorners(corners) {
+            const overlayCtx = overlayCanvas.getContext("2d");
+            clearOverlayCtx();
+
+            overlayCtx.beginPath();
+            overlayCtx.strokeStyle = "blue";
+            overlayCtx.lineWidth = 3;
+
+            // [x1,y1,x2,y2,x3,y3,x4,y4]
+            overlayCtx.moveTo(corners[0], corners[1]);
+            overlayCtx.lineTo(corners[2], corners[3]);
+            overlayCtx.lineTo(corners[4], corners[5]);
+            overlayCtx.lineTo(corners[6], corners[7]);
+            overlayCtx.lineTo(corners[0], corners[1]);
+
+            overlayCtx.stroke();
+        }
+
         function transformElem(h, elem) {
             // column major order
             let transform = [h[0], h[3], 0.0, h[6],

diff --git a/src/Workers/WebARKitCVWorkers.ts b/src/Workers/WebARKitCVWorkers.ts
@@ -91,6 +91,7 @@ export class WebARKitCVOrbWorker extends AbstractWebARKitCVWorker {
    */
   public found(msg: any) {
     let world: Float64Array;
+    let corners: any;
     if (!msg) {
       // commenting out this routine see https://github.com/webarkit/ARnft/pull/184#issuecomment-853400903
       //if (world) {
@@ -102,8 +103,9 @@ export class WebARKitCVOrbWorker extends AbstractWebARKitCVWorker {
       //}
     } else {
       world = JSON.parse(msg.matrix);
+      corners= JSON.parse(msg.corners);
       const matrixEvent = new CustomEvent<object>("getMatrix", {
-        detail: { matrix: world },
+        detail: { matrix: world, corners: corners },
       });
       this.target.dispatchEvent(matrixEvent);
     }

diff --git a/src/core/WebARKitCoreCV.ts b/src/core/WebARKitCoreCV.ts
@@ -8,6 +8,7 @@ export class WebARKitCoreCV {
   private template_keypoints_vector: any;
   private template_descriptors: any;
   private corners: any;
+  private corners_out: any;
   private listeners: object;
   private ValidPointTotal = 15;
   private N = 10.0;
@@ -49,10 +50,11 @@ export class WebARKitCoreCV {
     let refCols = msg.trackableWidth;
 
     let mat = new this.cv.Mat(refRows, refCols, this.cv.CV_8UC4);
+    let grayImage = new this.cv.Mat(refRows, refCols, this.cv.CV_8UC1);
 
     mat.data.set(src.data);
 
-    this.cv.cvtColor(mat, mat, this.cv.COLOR_RGBA2GRAY, 0);
+    this.cv.cvtColor(mat, grayImage, this.cv.COLOR_RGBA2GRAY, 0);
 
     let ksize = new this.cv.Size(this.BlurSize, this.BlurSize);
     let anchor = new this.cv.Point(-1, -1);
@@ -66,7 +68,7 @@ export class WebARKitCoreCV {
     let orb = new this.cv.ORB(10000);
 
     orb.detectAndCompute(
-      mat,
+      grayImage,
       noArray,
       this.template_keypoints_vector,
       this.template_descriptors,
@@ -153,7 +155,7 @@ export class WebARKitCoreCV {
 
       if (point.distance < 0.7 * point2.distance) {
         var frame_point = frame_keypoints_vector.get(point.queryIdx).pt;
-        console.log(frame_point.x)
+
         //frame_keypoints.push(frame_point);
         frame_keypoints.push(frame_point.x);
         frame_keypoints.push(frame_point.y);
@@ -167,7 +169,6 @@ export class WebARKitCoreCV {
         template_keypoints.push(template_point.y);
       }
     }
-    console.log(template_keypoints)
 
     var frameMat = new this.cv.Mat(frame_keypoints.length, 1, this.cv.CV_32FC2);
     var templateMat = new this.cv.Mat(
@@ -200,14 +201,15 @@ export class WebARKitCoreCV {
       console.log(valid)
 
       if (this.homographyValid(homography) == true) {
-          console.log('true')
           var out = this.fill_output(homography, valid);
           console.log("output from", out);
       }
       //this.homography_transform = homography.data64F;
-      this.homography_transform = out.slice(0,9)
+      this.homography_transform = out.slice(0,9);
+      this.corners_out = out.slice(10, 18);
     } else {
       this.homography_transform = null;
+      this.corners_out = null;
     }
 
     noArray.delete();
@@ -227,22 +229,21 @@ export class WebARKitCoreCV {
     result = {
       type: "found",
       matrix: JSON.stringify(this.homography_transform),
+      corners: JSON.stringify(this.corners_out),
     };
     return result;
   }
 
   homographyValid(H: any) {
-    console.log(H.floatAt(0,0))
     const det =
       H.doubleAt(0, 0) * H.doubleAt(1, 1) - H.doubleAt(1, 0) * H.doubleAt(0, 1);
       //H.floatAt(0, 0) * H.floatAt(1, 1) - H.floatAt(1, 0) * H.floatAt(0, 1);
-
     return 1 / this.N < Math.abs(det) && Math.abs(det) < this.N;
   }
 
   fill_output = (H: any, valid: boolean) => {
     let output = new Float64Array(17);
-    var warped = new this.cv.Mat(2, 2, this.cv.CV_64FC2);
+    let warped = new this.cv.Mat(2, 2, this.cv.CV_64FC2);
     this.cv.perspectiveTransform(this.corners, warped, H);
 
     output[0] = H.doubleAt(0, 0);
@@ -266,8 +267,8 @@ export class WebARKitCoreCV {
 
     console.log(output);
 
-    // corners.delete();
-    // warped.delete();
+    H.delete();
+    warped.delete();
 
     return output;
   };

diff --git a/types/src/core/WebARKitCoreCV.d.ts b/types/src/core/WebARKitCoreCV.d.ts
@@ -5,6 +5,7 @@ export declare class WebARKitCoreCV {
     private template_keypoints_vector;
     private template_descriptors;
     private corners;
+    private corners_out;
     private listeners;
     private ValidPointTotal;
     private N;
@@ -16,6 +17,7 @@ export declare class WebARKitCoreCV {
     track(msg: any): {
         type: string;
         matrix: string;
+        corners: string;
     };
     homographyValid(H: any): boolean;
     fill_output: (H: any, valid: boolean) => Float64Array;