source: level2/competencies/FacET/src/face_regions.cpp @ 234

Revision 74, 16.7 KB checked in by nik, 11 years ago (diff)

FacET import

Line 
1/* $Id: face_regions.cpp 5 2009-03-12 22:30:56Z mw $
2 * +-------------------------------------------------------------------+
3 * | This file contains parts of code from the application created for |
4 * | the Master thesis supervised by Marek Wnuk (Wroclaw University of |
5 * | Technology):  "Wykorzystanie systemu wizyjnego do rozpoznawania   |
6 * | emocji czlowieka" ("Vision system in human emotions recognition") |
7 * | by Marcin Namysl in June 2008.                                    |
8 * +-------------------------------------------------------------------+
9 *
10 * \author Marek Wnuk
11 * \date 2009.03.03
12 * \version 1.00.00
13 */
14
15/*
16    FacET is a library for detecting and parameterising face components.
17    Copyright (C) 2009  Marek Wnuk <marek.wnuk@pwr.wroc.pl>
18
19    This program is free software: you can redistribute it and/or modify
20    it under the terms of the GNU General Public License as published by
21    the Free Software Foundation, either version 3 of the License, or
22    (at your option) any later version.
23
24    This program is distributed in the hope that it will be useful,
25    but WITHOUT ANY WARRANTY; without even the implied warranty of
26    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
27    GNU General Public License for more details.
28
29    You should have received a copy of the GNU General Public License
30    along with this program.  If not, see <http://www.gnu.org/licenses/>.
31*/
32
33/*! \file face_regions.cpp
34 *
35 * The file defines a set of functions for face subregions (mouth and eyes)
36 * detection using gradient projection method.
37 *
38 * It contains FaceComponent class definition, which can be used to describe
39 * a single face component (mouth, eyes, eyebrows, nose, forehead).
40 * The FFace class is defined, which contains objects of type FaceComponent 
41 * (mouth, eyes, eyebrows, forehead, eyelids).
42 *
43 */
44
45#include <math.h>
46
47#include "facet.h"
48
49void FFace::clearElements()
50{
51  mouth.clear();
52  left_eye.clear();
53  right_eye.clear();
54  left_iris.clear();
55  right_iris.clear();
56  left_brow.clear();
57  right_brow.clear();
58  left_lid.clear();
59  right_lid.clear();
60  lip.clear();
61  nose.clear();
62  forehead.clear();
63}
64
65
66
67void counting(CvMat *matrix, int tab[MAX_SIZE], proj_type type,
68               count_mode mode, int start, int end)
69{
70  if (type==PROJ_HORIZONTAL)
71    for (int i=0; i < matrix->rows; i++){
72      tab[i]=0;
73      for (int j=start; j < end ; j++){
74        if (mode==CNT_MODE_BINARY){
75          tab[i]+= (cvGetAt(matrix,i,j).val[0]>0) ? 1 : 0;
76        }
77        else if (mode==CNT_MODE_NORMAL){
78          tab[i]+=(int)cvGetAt(matrix,i,j).val[0];       
79        }
80      }
81    }
82  else if (type==PROJ_VERTICAL)
83    for (int i=0; i < matrix->cols; i++){
84      tab[i]=0;
85      for (int j=start; j < end ; j++){
86        if (mode==CNT_MODE_BINARY){
87          tab[i]+= (cvGetAt(matrix,j,i).val[0]>0) ? 1 : 0;
88        }
89        else if (mode==CNT_MODE_NORMAL){
90          tab[i]+=(int)cvGetAt(matrix,j,i).val[0];       
91        }
92      }
93    }
94}
95
96
97void drawProjection(IplImage *img, int tab[MAX_SIZE], int start, int end,
98                    proj_type type, CvScalar color)
99{
100  CvPoint pt1 = {0,0};
101  CvPoint pt2 = {0,0};
102  for (int i=start+1; i < end; i++){
103    if (type==PROJ_VERTICAL){
104      pt1 = cvPoint( i-1, img->height - (tab[i-1] - 1)/70 );//300
105    pt2 = cvPoint( i, img->height - (tab[i] - 1)/70 );
106    }
107    else if (type==PROJ_HORIZONTAL){
108      pt1 = cvPoint((tab[i-1]-1)/50,i-1);
109      pt2 = cvPoint((tab[i]-1)/50,i);
110    }
111    else {
112      cerr << "Error in function 'drawProjection()\n";
113      cerr << "(Illegal projection type).\n";
114    }
115    cvLine(img,pt1,pt2,color,1,8);
116  }
117}
118
119
120int horizEyePosition(int tab[MAX_SIZE], int upperb, int lowerb)
121{
122  int max_horiz=0;
123  int max_val;
124
125  for (int i=lowerb; i < upperb; i++){
126    if (tab[i]>max_horiz){
127      max_horiz = tab[i];
128      max_val = i;
129    }
130  }
131
132  int second_max=0;
133  int max_val2;
134
135  for (int i=lowerb; i < upperb; i++) {
136    if ( (tab[i]>second_max) && (tab[i]<=max_horiz) &&
137         (abs(max_val-i)>(upperb-lowerb)/4) ) {
138      second_max = tab[i];
139      max_val2 = i;
140    }
141  }
142  return max(max_val, max_val2) - cvRound(abs(max_val2-max_val))/2;
143}
144
145
146int horizMouthPosition(int tab[MAX_SIZE], int img_height, int upperb)
147{
148  int max_horiz=0;
149  int ret_val = 0;
150  for (int i=upperb; i < img_height; i++){
151    if (tab[i]>max_horiz){
152      max_horiz = tab[i];
153      ret_val = i;
154    }
155  }
156  return ret_val;
157}
158
159
160void vertEyeBorder(int tab[MAX_SIZE], int img_width, int &ind_b, int &ind_e)
161{
162  int max_b, max_e;
163  max_b=max_e=0;
164
165  for (int i=0; i < img_width; i++){
166    if ( (i<=img_width/2) && (tab[i]>max_b) ){
167      max_b = tab[i];
168      ind_b = i;
169    }
170    if ( (i>img_width/2) && (tab[i]>max_e) ){
171      max_e = tab[i];
172      ind_e = i;
173    }   
174  }
175}
176
177double rint(double x) //CHANGE added rint function GC
178{
179  //middle value point test
180  if (ceil(x+0.5) == floor(x+0.5))
181    {
182      int a = (int)ceil(x);
183      if (a%2 == 0)
184        {return ceil(x);}
185      else
186        {return floor(x);}
187    }
188 
189  else return floor(x+0.5);
190}
191
192
193int vertEyeCenter(int tab[MAX_SIZE], int img_width, int offset)
194{
195  int max_vert=0;
196  int ret_val = 0;
197  for (int i=0; i < img_width; i++){
198
199    tab[i]=(int)rint(tab[i]/(2*offset)); //CHANGE round function GC
200    //tab[i]=(int)round(tab[i]/(2*offset));
201
202    if ( (tab[i]>max_vert) && (i>img_width/4) && (i<3*img_width/4) ){
203      max_vert = tab[i];
204      ret_val = i;
205    }
206  }
207  return ret_val;
208}
209
210
211void vertMouthBorder(int tab[MAX_SIZE], int width, int start, int end,
212                     int &lower_border, int &upper_border)
213{
214  int cntr=start;         // tab elements counter
215  int count=0;            // range elements counter
216  bool find_start=true;   // start of range searching mode
217  bool find_end=false;    // end of range searching mode
218
219  int range=(int)rint(width/10); //CHANGE round GC
220  //int range=(int)round(width/10);
221
222  // lower bound of the possible vertical mouth projection
223
224  upper_border=lower_border=start;
225
226  while (cntr++ < end){
227    if (find_start){
228      if (tab[cntr]>=100) count++;
229      else
230        if (count<=range) count = 0;
231        else {
232          lower_border=cntr-count;
233          upper_border=cntr;
234          find_end=true;
235          find_start=false;
236          count=0;
237        }
238    }
239    if (find_end){
240      if (tab[cntr]>=100) count++;
241      else
242        if (count<=range) count = 0;
243        else {
244          upper_border=cntr;
245          count=0;
246        }
247    }
248  }
249}
250
251
252double eyeMatching(IplImage *img, IplImage *templ, CvPoint &topLeft)
253{
254  vector<CvPoint> foundPointsList;
255  vector<double> confidencesList;
256
257  if( (templ->width <= img->width) || (templ->height <= img->height) ) {
258    topLeft.x = 0;
259    topLeft.y = 0;
260    cerr << "ERROR: Fast match template failed (size(temp)>size(img))\n";
261    return 0;
262  }
263
264  else if(!FastMatchTemplate(*img, *templ, &foundPointsList, &confidencesList,
265                        90, false))
266    cerr << "\nERROR: Fast match template failed (< 90%).\n";
267 
268  int numPoints = foundPointsList.size();
269
270  double max_confidence = 90;
271
272
273  for( int currPoint = 0; currPoint < numPoints; currPoint++ )
274  {
275    const CvPoint& point = foundPointsList[currPoint];
276   
277    if ( confidencesList[currPoint] >= max_confidence ) { //*****
278
279      max_confidence = confidencesList[currPoint];
280     
281      CvSize size = cvGetSize( templ );
282
283      topLeft.x = point.x - size.width / 2;
284      topLeft.y = point.y - size.height / 2;
285    }
286  }
287
288  if (numPoints==0){
289    topLeft.x = 0;
290    topLeft.y = 0;
291  }
292
293  return max_confidence;
294}
295
296
297void detectFacialRegions(IplImage *src,  FaceComponent &eye_left,
298                         FaceComponent &eye_right, FaceComponent &mouth,
299                         int &width, bool detect_mouth)
300{
301  IplImage *tmp  = 0;  // tmp image
302  IplImage *grad = 0;  // gradient image
303  IplImage *img  = 0;  // gradient with projections and boudaries
304
305  int* ver_val = new int [src->width];  //CHANGE GC
306  // int ver_val[src->width];//values for vertical projection
307  int* hor_val = new int [src->height];
308  // int hor_val[src->height];//values for horizontal projection
309
310  int x[9] = {0,1,0,1,1,1,0,1,0};
311
312  int EHC, FC, MHC;
313  int EV1, EV2, EH1, EH2;
314  int MV1, MV2, MH1, MH2;
315
316  int lowerbound=1*src->height/4; // eyes search region boudaries
317  int upperbound=3*src->height/6;
318
319  int offset=src->height/4;
320
321  int off=(int)rint(min(src->width,src->height)/50); //CHANGE round GC
322  // int off=(int)round(min(src->width,src->height)/50);
323
324  int sum,count,average;
325 
326  tmp  = cvCreateImage(cvGetSize(src),src->depth,src->nChannels);
327  cvCopy(src,tmp);
328
329  img  = cvCreateImage(cvGetSize(src),src->depth,src->nChannels);
330  grad = cvCreateImage(cvGetSize(src), src->depth, 1);
331 
332  // calculate gradient of the source image
333  cvSmooth(src,src,3,3,1,1);
334 
335  IplConvKernel *kernel = cvCreateStructuringElementEx(3, 3, 1, 1,
336                                                       CV_SHAPE_CUSTOM, x);
337  cvMorphologyEx(src, img, tmp, kernel, CV_MOP_GRADIENT, 1); //gradient image
338  cvReleaseStructuringElement( &kernel );
339
340  cvCvtColor(img, grad, CV_RGB2GRAY);
341
342  //CvMat *matc1 = 0; // gradient image matrix
343  //CvMat *matc3 = 0; // source image matrix
344  CvMat mat_temp;
345
346  //matc1 = cvCreateMat(grad->height, grad->width, CV_8UC1);
347  //matc3 = cvCreateMat(src->height, src->width, CV_8UC3);
348
349  cvGetMat(grad,&mat_temp);//get gradient image matrix //***
350
351  /////////////// Operators for finding the eyes region //////////////////
352   
353  counting(&mat_temp,ver_val,PROJ_VERTICAL,CNT_MODE_NORMAL,0,src->height);
354 
355  // vertical eyes boudaries
356  vertEyeBorder(ver_val,src->width,EV1,EV2);
357
358  width = EV2 - EV1; // eststimated face width
359
360  if (EV1<1) EV1=1;
361  if (EV2>src->height) EV2=src->height;
362   
363  counting(&mat_temp,hor_val,PROJ_HORIZONTAL,CNT_MODE_NORMAL,EV1+2*off,EV2-2*off);
364
365  // horizontal eyes location
366  EHC = horizEyePosition(hor_val,upperbound,lowerbound);
367
368  if (EHC<1) EHC=1;
369  if (EHC>img->height) EHC=img->height;
370
371  EH1 = EHC-(int)rint(0.5*offset); //CHANGE round GC
372  EH2 = EHC+(int)rint(0.8*offset);
373  // EH1 = EHC-(int)round(0.5*offset);
374  // EH2 = EHC+(int)round(0.8*offset);
375
376  if (EH1<1) EH1=1;
377  if (EH2>src->height) EH2 = src->height;
378 
379  cvGetMat(src, &mat_temp); // get source image matrix
380
381  counting(&mat_temp,ver_val,PROJ_VERTICAL,CNT_MODE_NORMAL,
382            EH1,EH2);
383
384  // find vertical eye center
385  FC = vertEyeCenter(ver_val,src->width,offset);
386
387  if (FC<1) FC=1;
388  if (FC>img->height) FC = img->height;
389
390  // vertical projection - find the face center
391  // coordinates of eyes bounding rectangles
392  eye_left.p1=cvPoint(EV1, EH1);
393  eye_left.p2=cvPoint(FC-off, EH2);
394
395  eye_right.p1=cvPoint(FC+off, EH1-(int)rint(0.3*offset)); //CHANGE round GC
396  // eye_right.p1=cvPoint(FC+off, EH1-(int)round(0.3*offset));
397
398  eye_right.p2=cvPoint(EV2, EH2);
399 
400  if (detect_mouth) {
401    /////////////// Operators for finding the mouth region /////////////////
402
403    int mouthbound = 3*src->height/4;
404    MHC = horizMouthPosition(hor_val, src->height, mouthbound);
405
406    if (MHC<1) MHC=1;
407    if (MHC>img->height) MHC = img->height;
408
409    MH1 = MHC-(int)(0.4*offset);
410    //MH2 = MHC+(int)(0.8*offset);
411    MH2 = src->height;
412
413    if (MH1<1) MH1 = 1;
414    if (MH2>src->height) MH2 = src->height;
415
416    cvGetMat(grad,&mat_temp,0); // get gradient image matrix
417 
418    // vertical projection in the mouth region
419    counting(&mat_temp,ver_val,PROJ_VERTICAL,CNT_MODE_NORMAL,
420              MH1,MH2);
421
422    sum=count=average=0;
423
424    for (int i=eye_left.p1.x; i < eye_right.p2.x; i++) sum+=ver_val[i];
425    count = eye_right.p2.x - eye_left.p1.x;
426
427    average = (int)rint(sum/count); //CHANGE round GC
428    // average = (int)round(sum/count);
429
430    for (int i=eye_left.p1.x; i < eye_right.p2.x; i++){
431      if ((ver_val[i]>average/2) && (i>src->width/4) && (i<3*src->width/4)){
432        ver_val[i]=100;
433      }
434      else ver_val[i]=10;
435    }
436 
437    // vertical mouth projection drawing
438    // vertical boundaries for the mouth region
439    vertMouthBorder(ver_val, src->width, eye_left.p1.x, eye_right.p2.x,
440                    MV1, MV2);
441
442    // mouth limits relaxation
443    if (MV1<=0) MV1=FC;
444    if (MV2>=src->width) MV2=FC;
445   
446    MV1-=offset/5;
447    MV2+=offset/5;
448
449    // coordinates of mouth bounding box
450    mouth.p1 = cvPoint(MV1,MH1); 
451    mouth.p2 = cvPoint(MV2,MH2);
452  }
453
454  cvReleaseImage( &img );
455  cvReleaseImage( &tmp );
456  cvReleaseImage( &grad );
457
458  //  cvReleaseMat( &matc1 );
459  //  cvReleaseMat( &matc3 );
460
461  delete [] ver_val;    //CHANGE delete allocated memory GC
462  delete [] hor_val;
463
464}
465
466double eyeDetection(IplImage *eye_img, IplImage *templ, int correction,
467                    FaceComponent &iris)
468{
469  int w,h;
470  /*
471  h=(int)ceil(width/10);
472  w=2*h;
473  */
474 
475  h=static_cast<int>(ceil((float)eye_img->width/14));
476  w=static_cast<int>(ceil((float)eye_img->height/6));
477
478  if (eye_img->width <= w ) w++;
479  if (eye_img->height <= h ) h++;
480 
481  IplImage *scale = cvCreateImage(cvSize(w, h),
482                                  IPL_DEPTH_8U,1 );
483 
484  cvResize(templ,scale);
485  //IplImage *scale = cvCreateImage(cvGetSize(templ),8,1);
486  //cvCopy(templ,scale);
487
488  if ((eye_img->width) <= 0) {
489    iris = FaceComponent(cvPoint(0,0),cvPoint(0,0));
490    return 0;
491  }
492
493  IplImage *img = cvCreateImage(cvGetSize(eye_img), 8, 1);
494
495  cvCvtColor(eye_img, img, CV_RGB2GRAY);
496
497  CvPoint iris_pt1;
498
499  double confidence = eyeMatching(img, scale, iris_pt1);
500
501  iris_pt1 = cvPoint(iris_pt1.x,iris_pt1.y-correction);
502
503  CvPoint iris_pt2 = cvPoint(iris_pt1.x+scale->width-1,
504                             iris_pt1.y+scale->height-1);
505
506  iris = FaceComponent(iris_pt1,iris_pt2);
507
508  cvReleaseImage( &img );
509  cvReleaseImage( &scale );
510
511  return confidence;
512}
513
514
515void checkSizeOczu(FaceComponent &el1, FaceComponent &el2)
516{
517  // Check the eye regions size constraints
518 
519  if (el1.p1.x > el1.p2.x) {
520    int tmp = 0;
521    tmp = el1.p1.x;
522    el1.p1.x = el1.p2.x;
523    el1.p2.x = tmp;
524  }
525  else if (el1.p1.x == el1.p2.x)
526    el1.p2.x += 1;
527 
528  if (el1.p1.y > el1.p2.y) {
529    int tmp = 0;
530    tmp = el1.p1.y;
531    el1.p1.y = el1.p2.y;
532    el1.p2.y = tmp;
533  }
534  else if (el1.p1.y == el1.p2.y)
535    el1.p2.y += 1;
536 
537  if (el2.p1.x > el2.p2.x) {
538    int tmp = 0;
539    tmp = el2.p1.x;
540    el2.p1.x = el2.p2.x;
541    el2.p2.x = tmp;
542  }
543  else if (el2.p1.x == el2.p2.x)
544    el2.p2.x += 1;
545 
546  if (el2.p1.y > el2.p2.y) {
547    int tmp = 0;
548    tmp = el2.p1.y;
549    el2.p1.y = el2.p2.y;
550    el2.p2.y = tmp;
551  }
552  else if (el2.p1.y == el2.p2.y)
553    el2.p2.y += 1;
554}
555
556void findFaceRegions(IplImage *source_img, FFace &face, bool detect_mouth,
557                           bool detect_eyeballs)
558{
559  FaceComponent eye_right, eye_left, mouth;
560
561  IplImage *result_img = cvCreateImage(cvGetSize(source_img),source_img->depth,
562                                       source_img->nChannels);
563  cvCopy(source_img,result_img);
564
565  detectFacialRegions(result_img, eye_left, eye_right, mouth, face.width,
566                      detect_mouth);
567
568  checkSizeOczu(eye_left, eye_right);
569
570  FaceComponent liris, riris;
571
572  int lo_p1y=eye_left.p1.y;
573  int po_p1y=eye_right.p1.y;
574  int correction = 0;
575
576  if (detect_eyeballs) {
577    IplImage *templ_l = 0;
578    IplImage *templ_r = 0;
579    IplImage *leye_img = NULL;
580    IplImage *reye_img = NULL;
581
582    // eye_left.p1.y contains lower value of Y coord. (???)
583    // eye_right.p1.y greater (???)
584   
585    int diff_leye_x = eye_left.p2.x - eye_left.p1.x;
586    int diff_peye_x = eye_right.p2.x - eye_right.p1.x;
587
588    if (diff_leye_x<0) eye_left.p1.x = eye_left.p2.x;
589    if (diff_peye_x<0) eye_right.p1.x = eye_right.p2.x;
590   
591    correction = po_p1y - lo_p1y;
592
593    double confid_l, confid_r;
594 
595    templ_l = cvLoadImage("input/leye.jpg", 0); // MW: to be improved
596    templ_r = cvLoadImage("input/peye.jpg", 0);
597
598    cvSetImageROI(result_img, cvRect(eye_left.p1.x, eye_left.p1.y,
599                                     eye_left.p2.x-eye_left.p1.x,
600                                     eye_left.p2.y-eye_left.p1.y));
601    confid_l = eyeDetection(result_img, templ_l, correction, liris);
602    cvResetImageROI(result_img);
603
604    cvSetImageROI(result_img, cvRect(eye_right.p1.x, eye_right.p1.y,
605                                     eye_right.p2.x-eye_right.p1.x,
606                                     eye_right.p2.y-eye_right.p1.y));
607    confid_r = eyeDetection(result_img, templ_r, correction, riris);
608    cvResetImageROI(result_img);
609
610    cvReleaseImage( &templ_r );
611    cvReleaseImage( &templ_l );
612    cvReleaseImage( &leye_img );
613    cvReleaseImage( &reye_img );
614  }
615
616  cvReleaseImage( &result_img );
617
618  eye_left.p1.y = po_p1y;
619
620  face.left_eye = eye_left;
621  face.right_eye = eye_right;
622 
623  if (detect_mouth)  face.mouth = mouth; 
624
625  if (detect_eyeballs) {
626
627    riris.p1.y += correction;
628    riris.p2.y += correction;
629
630    face.left_iris = liris;
631    face.right_iris = riris;
632
633    face.left_iris.p1 = cvPoint(face.left_iris.p1.x+face.left_eye.p1.x,
634                                face.left_iris.p1.y+face.left_eye.p1.y);
635    face.left_iris.p2 = cvPoint(face.left_iris.p2.x+face.left_eye.p1.x,
636                                face.left_iris.p2.y+face.left_eye.p1.y);
637    face.right_iris.p1=cvPoint(face.right_iris.p1.x+face.right_eye.p1.x,
638                               face.right_iris.p1.y+face.right_eye.p1.y);
639    face.right_iris.p2=cvPoint(face.right_iris.p2.x+face.right_eye.p1.x,
640                               face.right_iris.p2.y+face.right_eye.p1.y);
641  }
642}
643
644/*
645void example_usage(char *twarz)
646{
647  IplImage *input_img = NULL;
648
649  FFace face;
650
651  input_img = cvLoadImage( twarz, -1 );
652
653  findFaceRegions(input_img, face);
654 
655  //I5 cvNamedWindow( "Source", 1 );
656  //I5 cvShowImage( "Source", input_img );
657  //I5 cvDestroyWindow("Source");
658  cvWaitKey(0);
659
660  cvReleaseImage( &input_img );
661}
662*/
Note: See TracBrowser for help on using the repository browser.