diff --git a/README.md b/README.md index a7fb9a0..474c102 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,19 @@ module.exports = { The TypeScript type declarations may not be up to date with the latest OpenCV.js. Refer to [cvKeys.json](doc/cvKeys.json) to check the available methods and properties at runtime. +## OpenCV Contrib Modules + +The default opencv.js build includes core OpenCV modules but **does not include opencv_contrib modules** (like the `face` module). + +If you need opencv_contrib modules such as: +- `cv.face.createFacemarkLBF()` - Facial landmark detection +- Other face analysis features +- Extended features from opencv_contrib + +You will need to build a custom opencv.js from source. See the [Face Module Guide](doc/FACE_MODULE.md) for detailed instructions. + +**Available modules in default build:** calib3d, core, dnn, features2d, flann, imgproc, js, objdetect, photo, video + # Star History [![Star History Chart](https://api.star-history.com/svg?repos=techstark/opencv-js&type=Date)](https://star-history.com/#techstark/opencv-js&Date) diff --git a/doc/FACE_MODULE.md b/doc/FACE_MODULE.md new file mode 100644 index 0000000..3a6098d --- /dev/null +++ b/doc/FACE_MODULE.md @@ -0,0 +1,183 @@ +# Face Module Support - createFacemarkLBF + +## Overview + +The `face` module in OpenCV provides facial landmark detection capabilities, including the `createFacemarkLBF` function. This document explains how to use these features with opencv-js. + +## Important: Face Module Availability + +**The default opencv.js binary does NOT include the face module.** The face module is part of the opencv_contrib repository and requires a custom build. + +### Current opencv.js Build Includes: +- calib3d +- core +- dnn +- features2d +- flann +- imgproc +- js +- objdetect +- photo +- video + +### NOT Included (requires custom build): +- **face** (contains createFacemarkLBF) +- and other opencv_contrib modules + +## TypeScript Support + +This package provides TypeScript type definitions for the face module, even though it's not included in the default opencv.js binary. These type definitions are available for users who build custom opencv.js with the face module enabled. + +## Using createFacemarkLBF + +### Option 1: Build Custom opencv.js (Recommended) + +To use `cv.face.createFacemarkLBF`, you need to build opencv.js from source with opencv_contrib modules: + +1. **Clone OpenCV and OpenCV Contrib repositories:** + ```bash + git clone https://github.com/opencv/opencv.git + git clone https://github.com/opencv/opencv_contrib.git + ``` + +2. **Install Emscripten SDK:** + ```bash + git clone https://github.com/emscripten-core/emsdk.git + cd emsdk + ./emsdk install latest + ./emsdk activate latest + source ./emsdk_env.sh + ``` + +3. **Build opencv.js with face module:** + ```bash + cd opencv + python ./platforms/js/build_js.py build_js \ + --cmake_option="-DOPENCV_EXTRA_MODULES_PATH=../opencv_contrib/modules" \ + --build_wasm + ``` + +4. **Replace the opencv.js in your project** with the newly built one from `build_js/bin/opencv.js` + +### Option 2: Use Alternative Face Detection Methods + +If you don't need facial landmark detection specifically, consider using the built-in face detection methods: + +- **FaceDetectorYN** - Available in the default opencv.js build +- **CascadeClassifier** - Haar cascade face detection (available) + +Example using CascadeClassifier: +```typescript +import cv from "@techstark/opencv-js"; + +// Load face cascade classifier +const faceCascade = new cv.CascadeClassifier(); +faceCascade.load('haarcascade_frontalface_default.xml'); + +// Detect faces +const gray = new cv.Mat(); +cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY); + +const faces = new cv.RectVector(); +faceCascade.detectMultiScale(gray, faces); + +console.log(`Detected ${faces.size()} faces`); + +// Clean up +faces.delete(); +gray.delete(); +faceCascade.delete(); +``` + +## Example Usage (with custom build) + +Once you have opencv.js built with the face module, you can use it like this: + +```typescript +import cv from "@techstark/opencv-js"; + +async function detectFacialLandmarks() { + // Create facemark instance + const facemark = cv.face.createFacemarkLBF(); + + // Load pre-trained model + // Download from: https://github.com/kurnianggoro/GSOC2017/tree/master/data + facemark.loadModel('lbfmodel.yaml'); + + // First, detect faces using a face detector + const faceCascade = new cv.CascadeClassifier(); + faceCascade.load('haarcascade_frontalface_default.xml'); + + // Convert image to grayscale + const gray = new cv.Mat(); + cv.cvtColor(image, gray, cv.COLOR_RGBA2GRAY); + + // Detect faces + const faces = new cv.RectVector(); + faceCascade.detectMultiScale(gray, faces); + + // Detect landmarks for each face + const landmarks = new cv.MatVector(); + const success = facemark.fit(gray, faces, landmarks); + + if (success) { + // Process landmarks + for (let i = 0; i < landmarks.size(); i++) { + const points = landmarks.get(i); + // Each face has 68 landmark points (for LBF model) + console.log(`Face ${i} has ${points.rows} landmarks`); + + // Draw landmarks on image + for (let j = 0; j < points.rows; j++) { + const x = points.floatAt(j, 0); + const y = points.floatAt(j, 1); + cv.circle(image, new cv.Point(x, y), 2, new cv.Scalar(0, 255, 0), -1); + } + } + } + + // Clean up + landmarks.delete(); + faces.delete(); + gray.delete(); + faceCascade.delete(); + facemark.delete(); +} +``` + +## FacemarkLBF Parameters + +You can customize the FacemarkLBF algorithm with parameters: + +```typescript +const params = new cv.face.FacemarkLBF_Params(); +params.cascade_depth = 10; +params.tree_depth = 5; +params.num_trees_per_cascade_level = 500; +params.learning_rate = 0.1; + +const facemark = cv.face.createFacemarkLBF(params); +``` + +## Pre-trained Models + +To use FacemarkLBF, you need a pre-trained model file. You can download models from: + +- [OpenCV Face Module Models](https://github.com/kurnianggoro/GSOC2017/tree/master/data) +- The default LBF model: `lbfmodel.yaml` (~56MB) + +## Resources + +- [OpenCV Face Module Documentation](https://docs.opencv.org/4.x/d1/d1d/group__face.html) +- [FacemarkLBF Paper](https://www.cv-foundation.org/openaccess/content_cvpr_2014/papers/Ren_Face_Alignment_at_2014_CVPR_paper.pdf) +- [Building opencv.js](https://docs.opencv.org/4.x/d4/da1/tutorial_js_setup.html) +- [opencv_contrib Repository](https://github.com/opencv/opencv_contrib) + +## Summary + +1. **Type definitions are available** - This package includes TypeScript types for `cv.face.createFacemarkLBF` +2. **Runtime not available by default** - The face module is not in the default opencv.js build +3. **Custom build required** - To use face module at runtime, build opencv.js from source with opencv_contrib +4. **Alternatives exist** - Consider using FaceDetectorYN or CascadeClassifier if you only need face detection + +If you have questions or need help building opencv.js with the face module, please open an issue on GitHub. diff --git a/doc/face-example.ts b/doc/face-example.ts new file mode 100644 index 0000000..62a5069 --- /dev/null +++ b/doc/face-example.ts @@ -0,0 +1,76 @@ +/** + * Example TypeScript code demonstrating type definitions for cv.face.createFacemarkLBF. + * + * NOTE: This code will compile with TypeScript but will NOT run with the default opencv.js + * because the face module is not included in the default build. + * + * To run this code, you need to build opencv.js from source with opencv_contrib modules. + * See doc/FACE_MODULE.md for instructions. + */ + +import * as cv from "../src/types/opencv"; + +async function exampleFacialLandmarkDetection() { + // This example shows TypeScript type checking for face module + // The types are available, but runtime will fail unless using custom opencv.js build + + // Create facemark instance with default parameters + const facemark = cv.face.createFacemarkLBF(); + + // Or create with custom parameters + const params = new cv.face.FacemarkLBF_Params(); + params.cascade_depth = 10; + params.tree_depth = 5; + params.num_trees_per_cascade_level = 500; + const facemarkCustom = cv.face.createFacemarkLBF(params); + + // Load pre-trained model + facemark.loadModel("lbfmodel.yaml"); + + // Create face detector + const faceCascade = new cv.CascadeClassifier(); + faceCascade.load("haarcascade_frontalface_default.xml"); + + // Assume we have an image (this is pseudocode) + const image = new cv.Mat(); + const gray = new cv.Mat(); + cv.cvtColor(image, gray, cv.COLOR_RGBA2GRAY); + + // Detect faces + const faces = new cv.RectVector(); + faceCascade.detectMultiScale(gray, faces); + + // Fit landmarks + const landmarks = new cv.MatVector(); + const success: boolean = facemark.fit(gray, faces, landmarks); + + if (success) { + console.log(`Detected landmarks for ${landmarks.size()} faces`); + + // Process each face's landmarks + for (let i = 0; i < landmarks.size(); i++) { + const points = landmarks.get(i); + console.log(`Face ${i}: ${points.rows} landmark points`); + + // Draw landmarks (68 points for LBF model) + for (let j = 0; j < points.rows; j++) { + const x = points.floatAt(j, 0); + const y = points.floatAt(j, 1); + cv.circle(image, new cv.Point(x, y), 2, new cv.Scalar(0, 255, 0), -1); + } + } + } + + // Clean up + landmarks.delete(); + faces.delete(); + gray.delete(); + image.delete(); + faceCascade.delete(); + facemark.delete(); + facemarkCustom.delete(); +} + +// TypeScript type checking will pass for this file +// Runtime execution requires custom opencv.js build with face module +export { exampleFacialLandmarkDetection }; diff --git a/package-lock.json b/package-lock.json index 2507829..49506c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3321,10 +3321,11 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, + "license": "MIT", "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -7112,9 +7113,9 @@ "dev": true }, "js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "requires": { "argparse": "^1.0.7", diff --git a/src/types/opencv/BackgroundSubtractor.ts b/src/types/opencv/BackgroundSubtractor.ts index c8815ae..6df1a04 100644 --- a/src/types/opencv/BackgroundSubtractor.ts +++ b/src/types/opencv/BackgroundSubtractor.ts @@ -1,9 +1,15 @@ -import type { Algorithm, bool, double, InputArray, OutputArray } from "./_types"; +import type { + Algorithm, + bool, + double, + InputArray, + OutputArray, +} from "./_types"; /** * Base class for background/foreground segmentation algorithms. * - * The class is only used to define the common interface for the whole family of background/foreground + * The class is only used to define the common interface for the whole family of background/foreground * segmentation algorithms. * * Source: @@ -17,20 +23,24 @@ export declare class BackgroundSubtractor extends Algorithm { * * @param image Next video frame. * @param fgmask The output foreground mask as an 8-bit binary image. - * @param learningRate The value between 0 and 1 that indicates how fast the background model is learnt. + * @param learningRate The value between 0 and 1 that indicates how fast the background model is learnt. * Negative parameter value makes the algorithm use some automatically chosen learning rate. - * 0 means that the background model is not updated at all, 1 means that the background model is + * 0 means that the background model is not updated at all, 1 means that the background model is * completely reinitialized from the last frame. */ - public apply(image: InputArray, fgmask: OutputArray, learningRate?: double): void; + public apply( + image: InputArray, + fgmask: OutputArray, + learningRate?: double, + ): void; /** * Computes a background image. * * @param backgroundImage The output background image. * - * @note Sometimes the background image can be very blurry, as it contain the average background + * @note Sometimes the background image can be very blurry, as it contain the average background * statistics. */ public getBackgroundImage(backgroundImage: OutputArray): void; -} \ No newline at end of file +} diff --git a/src/types/opencv/BackgroundSubtractorMOG2.ts b/src/types/opencv/BackgroundSubtractorMOG2.ts index ebc9460..06496fc 100644 --- a/src/types/opencv/BackgroundSubtractorMOG2.ts +++ b/src/types/opencv/BackgroundSubtractorMOG2.ts @@ -3,7 +3,7 @@ import type { BackgroundSubtractor, bool, double, int } from "./_types"; /** * Gaussian Mixture-based Background/Foreground Segmentation Algorithm. * - * The class implements the Gaussian mixture model background subtraction described in [Zivkovic2004] + * The class implements the Gaussian mixture model background subtraction described in [Zivkovic2004] * and [Zivkovic2006]. * * Source: @@ -12,11 +12,15 @@ import type { BackgroundSubtractor, bool, double, int } from "./_types"; export declare class BackgroundSubtractorMOG2 extends BackgroundSubtractor { /** * @param history Length of the history. - * @param varThreshold Threshold on the squared Mahalanobis distance between the pixel and the model - * to decide whether a pixel is well described by the background model. This parameter does not + * @param varThreshold Threshold on the squared Mahalanobis distance between the pixel and the model + * to decide whether a pixel is well described by the background model. This parameter does not * affect the background update. - * @param detectShadows If true, the algorithm will detect shadows and mark them. It decreases the + * @param detectShadows If true, the algorithm will detect shadows and mark them. It decreases the * speed a bit, so if you do not need this feature, set the parameter to false. */ - public constructor(history?: int, varThreshold?: double, detectShadows?: bool); -} \ No newline at end of file + public constructor( + history?: int, + varThreshold?: double, + detectShadows?: bool, + ); +} diff --git a/src/types/opencv/_types.ts b/src/types/opencv/_types.ts index a3ff3c9..20eea36 100644 --- a/src/types/opencv/_types.ts +++ b/src/types/opencv/_types.ts @@ -48,3 +48,4 @@ export * from "./video_track"; export * from "./_hacks"; export * from "./Tracker"; export * from "./TrackerMIL"; +export * from "./face"; diff --git a/src/types/opencv/face.ts b/src/types/opencv/face.ts new file mode 100644 index 0000000..3bd9214 --- /dev/null +++ b/src/types/opencv/face.ts @@ -0,0 +1,232 @@ +import type { + Algorithm, + bool, + InputArray, + InputOutputArray, + Mat, + MatVector, + OutputArray, + Point2f, + Ptr, + Rect, + RectVector, +} from "./_types"; + +/** + * # Face Analysis + * ## Face landmark detection + * + * Facemark is a base class for all face landmark detection algorithms. + * It provides a unified interface for training and fitting face landmarks. + */ + +/** + * Abstract base class for all facemark models + * + * To use the Facemark API: + * 1. Create an instance using createFacemarkLBF() or other factory functions + * 2. Load a pre-trained model using loadModel() + * 3. Detect faces using a face detector (e.g., CascadeClassifier) + * 4. Fit the facemark model to detected faces using fit() + * + * All facemark models in OpenCV are supported using this unified API. + */ +export declare class Facemark extends Algorithm { + /** + * Loads a trained facemark model from file. + * + * @param model Path to the trained model file + */ + public loadModel(model: string): void; + + /** + * Detects facial landmarks on a face image. + * + * @param image Input image (grayscale or color) + * @param faces Vector of face rectangles (RectVector) detected by a face detector + * @param landmarks Output vector of matrices (MatVector) where each Mat contains 2D points (Point2f) + * representing facial landmarks for each detected face + * @returns true if landmarks were successfully detected, false otherwise + * + * @example + * ```typescript + * const faces = new cv.RectVector(); + * const landmarks = new cv.MatVector(); + * const success = facemark.fit(gray, faces, landmarks); + * if (success) { + * for (let i = 0; i < landmarks.size(); i++) { + * const points = landmarks.get(i); // Mat with Point2f data + * // points.rows = number of landmarks (e.g., 68 for LBF model) + * // points.cols = 2 (x, y coordinates) + * } + * } + * ``` + */ + public fit(image: InputArray, faces: RectVector, landmarks: MatVector): bool; +} + +/** + * FacemarkLBF - Local Binary Features based face landmark detector + * + * This is an implementation of the LBF (Local Binary Features) algorithm for + * facial landmark detection. It is fast and works well for real-time applications. + * + * Reference: + * Ren, S., Cao, X., Wei, Y., & Sun, J. (2014). + * Face alignment at 3000 fps via regressing local binary features. + * In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (pp. 1685-1692). + */ +export declare class FacemarkLBF extends Facemark { + /** + * Construct FacemarkLBF instance + */ + public constructor(); +} + +/** + * Parameters for FacemarkLBF training and detection + */ +export declare class FacemarkLBF_Params { + /** + * Shape offset multiplier for data augmentation + */ + public shape_offset: number; + + /** + * Number of cascades (stages) in the regression + */ + public cascade_depth: number; + + /** + * Number of trees per cascade + */ + public tree_depth: number; + + /** + * Depth of each tree + */ + public num_trees_per_cascade_level: number; + + /** + * Learning rate + */ + public learning_rate: number; + + /** + * Oversampling amount for training + */ + public oversampling_amount: number; + + /** + * Number of test coordinates for each tree node + */ + public num_test_coordinates: number; + + /** + * Lambda parameter for regularization + */ + public lambda: number; + + /** + * Number of test splits for each tree node + */ + public num_test_splits: number; + + /** + * Face detection configuration file (cascade classifier) + */ + public cascade_face: string; + + /** + * File containing facial feature points for training + */ + public model_filename: string; + + /** + * Flag for saving trained model + */ + public save_model: bool; + + /** + * Random seed for reproducibility + */ + public seed: number; + + /** + * Feature extraction radius + */ + public feats: InputArray; + + /** + * Pupil distance (for alignment) + */ + public pupils: InputArray; + + /** + * Create default parameters + */ + public constructor(); +} + +/** + * Face module namespace containing all face analysis functionality + */ +export declare namespace face { + /** + * Creates an instance of the FacemarkLBF algorithm. + * + * @param parameters Optional parameters for FacemarkLBF configuration + * @returns Pointer to the FacemarkLBF instance + * + * @example + * ```typescript + * // Create facemark instance + * const facemark = cv.face.createFacemarkLBF(); + * + * // Load pre-trained model + * facemark.loadModel('lbfmodel.yaml'); + * + * // Detect faces first (using CascadeClassifier) + * const faceCascade = new cv.CascadeClassifier(); + * faceCascade.load('haarcascade_frontalface_default.xml'); + * const faces = new cv.RectVector(); + * faceCascade.detectMultiScale(gray, faces); + * + * // Fit landmarks + * const landmarks = new cv.MatVector(); + * const success = facemark.fit(gray, faces, landmarks); + * + * if (success) { + * // Use landmarks + * for (let i = 0; i < landmarks.size(); i++) { + * const points = landmarks.get(i); + * // Process points... + * } + * } + * + * // Clean up + * landmarks.delete(); + * faces.delete(); + * facemark.delete(); + * faceCascade.delete(); + * ``` + */ + export function createFacemarkLBF( + parameters?: FacemarkLBF_Params, + ): FacemarkLBF; + + /** + * FacemarkLBF class + */ + export { FacemarkLBF }; + + /** + * FacemarkLBF parameters class + */ + export { FacemarkLBF_Params }; + + /** + * Base Facemark class + */ + export { Facemark }; +} diff --git a/src/types/opencv/imgproc_colormap.ts b/src/types/opencv/imgproc_colormap.ts index 010797f..75138ae 100644 --- a/src/types/opencv/imgproc_colormap.ts +++ b/src/types/opencv/imgproc_colormap.ts @@ -57,4 +57,4 @@ export declare const COLORMAP_CIVIDIS: ColormapTypes; // initializer: = 17 export declare const COLORMAP_TWILIGHT: ColormapTypes; // initializer: = 18 export declare const COLORMAP_TWILIGHT_SHIFTED: ColormapTypes; // initializer: = 19 export declare const COLORMAP_TURBO: ColormapTypes; // initializer: = 20 -export declare const COLORMAP_DEEPGREEN: ColormapTypes; // initializer: = 21 \ No newline at end of file +export declare const COLORMAP_DEEPGREEN: ColormapTypes; // initializer: = 21 diff --git a/test/face.test.ts b/test/face.test.ts new file mode 100644 index 0000000..543fa9a --- /dev/null +++ b/test/face.test.ts @@ -0,0 +1,39 @@ +import { setupOpenCv } from "./cv"; + +beforeAll(setupOpenCv); + +describe("Face module type definitions", () => { + it("should have face namespace and createFacemarkLBF type definitions", () => { + // This test verifies that the TypeScript type definitions are available + // Note: The actual face module is NOT included in the current opencv.js build + // These type definitions are provided for users who build custom opencv.js with face module + + // Type check: verify cv.face namespace exists in type definitions + const hasFaceNamespace = "face" in cv; + + // Since face module is not built into opencv.js, it won't exist at runtime + // But the type definitions should compile without errors + expect(hasFaceNamespace).toBe(false); + + // The following would work if face module was included in the build: + // const facemark = cv.face.createFacemarkLBF(); + // facemark.loadModel('lbfmodel.yaml'); + // const faces = new cv.RectVector(); + // const landmarks = new cv.MatVector(); + // facemark.fit(gray, faces, landmarks); + }); + + it("should document that face module requires custom opencv.js build", () => { + // This is a documentation test + // Users wanting to use cv.face.createFacemarkLBF need to: + // 1. Build opencv.js from source with opencv_contrib modules enabled + // 2. Include the face module in the build configuration + // 3. Use that custom opencv.js instead of the default one + + // The default opencv.js from docs.opencv.org includes: + // calib3d, core, dnn, features2d, flann, imgproc, js, objdetect, photo, video + // but NOT the face module + + expect(true).toBe(true); + }); +});