Enhanced facetracking for Processing, Openframeworks and Android
Tracking faces via openCV has been done on multiple occasions, but they are all limited to a certain sweetspot of about 1.5 meters from the camera. This is a consequence of a balance between the camera resolution and the haarcascade filter used. By resizing the resolution of the image used for tracking in realtime it is possible to trick the facetracker into tracking the face at up to 6 meters of distance. As a sideeffect the tracker will ignore other people standing nearby. In the following we will introduce a few snippets of code that solves this on multiple different platforms:
Warning the code are extractions of larger projects and therefore not perfectly tested and packaged. Since we have made multiple solutions for different platforms, it would simply be too tedious to write sample programs for all of them.
On Android via Processing
Since it is a rather complicated task to get openCV up and running on the Android platform, this solution takes advantage of the exsisting facetracking within Android camera api itself.
Download and install:
Limitations: This method is heavy on the cpu and has a low framerate
Created a new tab with the following code:
import ketai.cv.facedetector.*;
import ketai.camera.*;
PImage roiImage; // Declare variable "a" of type PImage
PImage myImage;
int MAX_FACES = 2;
float numFaces = 0;
KetaiCamera cam;
int largestFace = -1;
int largestFaceID = 0;
int cwidth =480;
int cheight = 320;
float roiX = 0;
float roiY = 0;
float roiScale = 0;
float lastRatio =0 ;
boolean facesinit = false;
boolean largetracking = true;
boolean bigModeOld = false;
KetaiSimpleFace[] faces = new KetaiSimpleFace[MAX_FACES];
KetaiSimpleFace[] largestFaceData = new KetaiSimpleFace[2];
boolean bigMode = true;
PGraphics rotatedImg;
PGraphics rotatedImg_small;
boolean readReady = false;
int qscale(int value, float scale)
{
return round(((float)value)/(scale));
}
void cam_setup() {
cam = new KetaiCamera(this, cwidth, cheight, 10);
rotatedImg = createGraphics(cwidth, cheight);
rotatedImg_small = createGraphics(cwidth/2, cheight/2);
rotatedImg.beginDraw();
rotatedImg.background(0);
rotatedImg.endDraw();
rotatedImg_small.beginDraw();
rotatedImg_small.background(0);
rotatedImg_small.endDraw();
cam.setCameraID(1);
cam.start();
faces = KetaiFaceDetector.findFaces(cam, MAX_FACES);
}
float xTmp = 0;
float yTmp =0;
float distTmp =0;
void cam_update(boolean drawDebug) {
fill(255);
ellipse(0, 0, 20, 20);
if (largestFaceData[1] != null && largestFaceData[0] != null)
{
if(drawDebug)
{
image(cam,0,0);
imageMode(CORNER);
pushMatrix();
translate(0, 320);
image(rotatedImg_small, 0, 0);
ellipse(largestFaceData[1].location.x, largestFaceData[1].location.y, 20, 20);
popMatrix();
ellipse(rotatedImg.width/2 +roiX*-1*1/roiScale, rotatedImg.height/2 +roiY*-1*1/roiScale, 20, 20);
}
}
noFill();
stroke(255, 0, 0);
strokeWeight(3);
if (readReady)
{
cam.read();
if (faces.length ==0)
{
bigMode = true;
rotatedImg.beginDraw();
rotatedImg.imageMode(CENTER);
rotatedImg.translate(rotatedImg.width/2, rotatedImg.height/2);
rotatedImg.rotate(PI);
rotatedImg_small.scale(1.2f, 1.2f);
rotatedImg.image(cam, 0, 0);
rotatedImg.imageMode(CORNER);
rotatedImg.endDraw();
faces = KetaiFaceDetector.findFaces(rotatedImg, MAX_FACES);
}
else
{
if (bigModeOld)
{
roiX = rotatedImg.width/2 -largestFaceData[0].location.x;
roiY = rotatedImg.height/2-largestFaceData[0].location.y;
roiScale = 1.0f;
}
else
{
roiX = roiX*0.3f + (rotatedImg_small.width/2-largestFaceData[1].location.x + roiX)*0.7f;
roiY = roiY*0.3f + (rotatedImg_small.height/2-largestFaceData[1].location.y + roiY)*0.7f;
roiScale = roiScale * 0.3f + 0.7f*(rotatedImg_small.width /largestFaceData[1].distance)/6; //zoom factor;
}
bigMode = false;
rotatedImg_small.beginDraw();
rotatedImg_small.background(0);
rotatedImg_small.translate(rotatedImg_small.width/2, rotatedImg_small.height/2);
rotatedImg_small.rotate(PI);
rotatedImg_small.translate(-roiX, -roiY);
rotatedImg_small.imageMode(CENTER);
rotatedImg_small.scale(roiScale, roiScale);
rotatedImg_small.image(cam, 0, 0, rotatedImg.width, rotatedImg.height);
//rotatedImg_small.ellipse(0, 0, 20, 20);
rotatedImg_small.endDraw();
faces = KetaiFaceDetector.findFaces(rotatedImg_small, 1);
numFaces = faces.length;
}
bigModeOld = bigMode;
largestFace= 0;
for (int i=0; i < faces.length; i++)
{
//We only get the distance between the eyes so we base our bounding box off of that
if (faces[i].distance > largestFace)
{
largestFace = (int)faces[i].distance;
largestFaceID = i;
}
}
if (faces.length > 0)
{
if (bigMode)
{
largestFaceData[0] = faces[largestFaceID];
}
else
{
largestFaceData[1] = faces[largestFaceID];
distTmp = distTmp * 0.5+ 0.5*faces[largestFaceID].distance * 10;
xTmp = xTmp * 0.5+ 0.5* rotatedImg.width/2 +roiX*-1*1/roiScale;
yTmp = yTmp * 0.5+ 0.5* rotatedImg.height/2 +roiY*-1*1/roiScale;
println("face: x:" + xTmp + " y:" +yTmp + " width:" +distTmp )
}
}
else
{
println("No face")
}
readReady = false;
}
}
int cam_refresh = 0;
void onCameraPreviewEvent()
{
readReady = true;
}
Use the code with the following commands:
void setup()
{
cam_setup();
}
void draw()
{
cam_update(true);
}
On Processing with openCV
This requires you to download the openCV library and Processing. Be aware that this library does not work well with the latest version of processing and an earlier version (2.0) may need to be downloaded.
The above example can be modified to be used with OpenCV and processing on windows and mac (download example here:
import hypermedia.video.*;
import processing.video.*;
import java.awt.Rectangle;
import java.awt.Point;
OpenCV opencv;
OpenCV opencv_small;
int contrast_value = 0;
int brightness_value = 0;
PImage roiImage; // Declare variable "a" of type PImage
PImage myImage;
PGraphics rotatedImg;
Capture cam;
int largestFace = -1;
int largestFaceID = 0;
int cwidth = 640;
int cheight = 480;
float roiX = 0;
float roiY = 0;
float roiHeight = 0;
float roiWidth = 0;
void setup() {
roiImage = new PImage(300, 300);
size(800, 600);
String[] cameras = Capture.list();
for (int i = 0; i < cameras.length; i++) {
println(cameras[i]);
}
cam = new Capture(this, cameras[4]);
cam.start();
opencv_small = new OpenCV( this );
opencv_small.allocate(300, 300);
opencv_small.cascade( OpenCV.CASCADE_FRONTALFACE_ALT );
opencv = new OpenCV( this );
opencv.allocate(800, 600);
opencv.cascade( OpenCV.CASCADE_FRONTALFACE_ALT );
println( "Drag mouse on X-axis inside this sketch window to change contrast" );
println( "Drag mouse on Y-axis inside this sketch window to change brightness" );
}
float faceX = 90;
float faceY = 90;
Rectangle[] faces;
Rectangle[] faces_small;
boolean facesinit = false;
boolean largetracking = true;
float lastRatio =0 ;
long timer = 0;
void draw() {
if (timer < millis())
{
timer = millis() +300;
if (cam.available() == true)
{
background(0);
cam.read();
if (largetracking)
{
if (facesinit && faces.length > 0)
{
roiX = round(faces[0].x-faces[0].width/2);
roiY = round(faces[0].y-faces[0].height/2);
roiWidth = faces[0].width *2;
roiHeight = roiWidth;// * ( roiImage.width / roiImage.height);
lastRatio = roiWidth /300 ;
roiImage.copy(cam,
(int)roiX,
(int)roiY,
(int)roiWidth, (int)roiHeight,
0, 0, roiImage.width, roiImage.height);
largetracking = false;
println("sdf");
}
}
else if (facesinit && faces_small.length > 0)
{
roiX =roiX + round(faces_small[0].x-faces_small[0].width/2) *0.5f ;
roiY = roiY + round(faces_small[0].y-faces_small[0].height/2) * 0.5f;
// float aa = lastWidth/faces_small[0].width;
roiWidth = roiWidth * 0.5f + faces_small[0].width*2*lastRatio*0.5f;
roiHeight =roiWidth;// roiWeidth; //roiWeidth * ( roiImage.width / roiImage.height);
lastRatio = roiWidth /300 ;
roiImage.copy(
cam,
(int) roiX,
(int)roiY,
(int)roiWidth, (int)roiWidth,
0, 0, roiImage.width, roiImage.height);
largetracking = false;
// println(roiWidth + " " + (roiImage.width-roiImage.height));
}
else
{
roiX = 0;
roiY = 0;
largetracking = true;
}
//DETECT
if (largetracking)
{
opencv.copy(cam);
opencv.convert( GRAY );
faces = opencv.detect( 1.2, 2, OpenCV.HAAR_DO_CANNY_PRUNING, 40, 40 );
for ( int i=0; i<faces.length; i++ ) {
rect( faces[i].x, faces[i].y, faces[i].width, faces[i].height );
}
}
image(cam, 0, 0);
opencv_small.threshold(254);
opencv_small.copy(roiImage);
opencv_small.brightness( brightness_value );
faces_small = opencv_small.detect( 1.2, 2, OpenCV.HAAR_DO_CANNY_PRUNING, 40, 40 );
translate(0, 300);
image(roiImage, 0, 0 );
//for ( int i=0; i<faces_small.length; i++ ) {
for ( int i=0; i<faces_small.length; i++ )
{
rect( faces_small[i].x, faces_small[i].y, faces_small[i].width, faces_small[i].height );
}
facesinit = true;
// DRAW
noFill();
stroke(255, 0, 0);
noFill();
stroke(255, 0, 0);
}
}
}
void mouseDragged() {
contrast_value = (int) map( mouseX, 0, width, -128, 128 );
brightness_value = (int) map( mouseY, 0, width, -128, 128 );
}
Alternatively one can use the ROI (Region of interest) implimentation - download example here:
class ZoneTracker
{
OpenCV opencv;
float zoneX;
float zoneY;
float zoneSize;
float targetX;
float targetY;
ZoneTracker(OpenCV o)
{
opencv=o;
zoneX=opencv.width/2;
zoneY=opencv.height/2;
zoneSize=200;
targetX=zoneX;
targetY=zoneY;
}
boolean track(PImage img)
{
opencv.copy(img);
opencv.ROI((int)(zoneX-zoneSize/2), (int)(zoneY-zoneSize/2), (int)zoneSize, (int)zoneSize);
opencv.convert( GRAY );
opencv.contrast( 30 );
opencv.contrast( 5 );
opencv.brightness( 5 );
Rectangle[] faces = opencv.detect( 1.2, 2, OpenCV.HAAR_DO_CANNY_PRUNING, 40, 40 );
if (faces.length>0)
{
// snapshot.copy(img, (int)(zoneX-zoneSize/2), (int)(zoneY-zoneSize/2), (int)zoneSize, (int)zoneSize, 0, 0, 300, 300);
float x=faces[0].x+faces[0].width/2 + (zoneX - zoneSize/2);
float y=faces[0].y+faces[0].height/2 + (zoneY - zoneSize/2);
float w=faces[0].width*2;
zoneX=zoneX*0.4+x*0.6;
zoneY=zoneY*0.4+y*0.6;
zoneSize=zoneSize*0.9+w*0.1;
targetX=x;
targetY=y;
return true;
}
else
{
zoneX=zoneX*0.95+(float)((img.width/2))*0.05;
zoneY=zoneY*0.95+(float)((img.height/2))*0.05;
targetX=targetX*0.9+(opencv.width/2)*0.1;
targetY=targetY*0.9+(opencv.height/2)*0.1;
zoneSize=zoneSize*0.95+200.0*0.05;
}
return false;
}
}
On openFrameworks with openCV
Here you need to install openFrameworks and ofxCV.
The core code looks as follows:
cam.update();
if(cam.isFrameNew()) {
Mat camMat = toCv(cam);
Mat croppedCamMat(camMat, roi);
resize(croppedCamMat, cropped);
cropped.update();
finder.update(cropped);
if(finder.size() > 0) {
cv::Rect roiTmp = toCv(finder.getObject(0));
roi.x = (roi.x * 0.9f + (roi.x+roiTmp.x-roiTmp.width/2.0f)*0.1f);
roi.y = (roi.y * 0.9f + (roi.y+roiTmp.y-roiTmp.height/2.0f)*0.1f);
roi.width = (roi.width * 0.9 + roiTmp.width/2*0.1f);
roi.height = (roi.height * 0.9 + roiTmp.height/2*0.1f);
/* roi.y = ofClamp(roi.y - roi.height/2,0,cam.height);
roi.width = roi.width * scaling;
roi.height = roi.height * scaling;
Mat camMat = toCv(cam);
Mat croppedCamMat(camMat, roi);
resize(croppedCamMat, cropped);*/
tracked = true;
trackerTimer = ofGetElapsedTimeMillis();
}
else
{
if(ofGetElapsedTimeMillis()-trackerTimer > 2000)
{
roi.x = roi.x * 0.99 + cam.width/6*2 * 0.01f;;
roi.y = roi.y * 0.99 + cam.height/6*2 * 0.01f;;
roi.width = roi.width * 0.95 + cam.width/3*0.05;
roi.height = roi.height * 0.95 + cam.height/3* 0.05;
}
else if(tracked)
{
roi.x = roi.x * 0.9 ;
roi.y = roi.y * 0.9 ;
roi.width = roi.width * 0.9 + cam.width*0.1;
roi.height = roi.height * 0.9 + cam.height* 0.1;
tracked = false;
}
}
roi.x = ofClamp(roi.x,0,cam.width);
roi.y = ofClamp(roi.y,0,cam.height);
roi.width = ofClamp(roi.width,0,cam.width-roi.x);
roi.height = ofClamp(roi.height,0,cam.height-roi.y);
}
x = roi.x + roi.width/2;
y = roi.y + roi.height/2;
width = roi.width;
An actual example of an implimentation can be found here