4.30 How to detect a bounding box for an object in cv::Mat of OpenCV?

As a simple application of OpenCV APIs, we consider to find a bounding box for an object in cv:Mat of OpenCV. In this example, we have applied the following OpenCV APIs and operations to cv::Mat to find a bounding box.

1. Apply cv::EdgePreseringFilter to an originalImage of cv::Mat format.

2. Apply cv::cvtColor to the the filtered Image to get a grayImage.

3. Apply cv::Canny Edge Detector to the grayImage.

4. Apply cv::FindContours to the edgeDetectedImage to find contours.

5. Apply cv::approxPolyDP to the contours to get approximate polygonal coordinates.

6. Get bounding rectangles from the contours polygonal coordinates.

7. Find a single maximum rectangle from the bounding rectanglese set.

For simplicity, we have not used NMS alogrithm in the operation 7 in this program.

The following BoundingBoxDetector is a simple example to implement above operations to detect a single bounding box for an object in cv::Mat image. In practive, this is a very primitive method, but it shows a decent result for an object with a simple background as shown bellow.










/*
 * BoundingBoxDetector.cpp 
 * Copyright (c) 2018 Antillia.com TOSHIYUKI ARAI. ALL RIGHTS RESERVED. 
 */

#define _CONSOLE_

#include <sol/ModuleFileName.h>
#include <sol/DropFiles.h>
#include <sol/LabeledTrackBar.h>
#include <sol/FileDialog.h>
#include <sol/StringT.h>
#include <sol/opencv/OpenCVApplicationView.h>
#include <sol/opencv/OpenCVImageView.h>


namespace SOL {

class MainView :public OpenCVApplicationView {

private:
  ////////////////////////////////////////////////////////////////////////////////////////
  //Inner class starts.
  class OriginalImageView :public OpenCVImageView {
  private:
    cv::Mat originalImage;
    
    cv::Mat& getMat()
    {
      return originalImage;
    }
    
    void display()
    {
      show(originalImage);
    }
    
  public:
    OriginalImageView(View* parent, const char* name, Args& args)
    :OpenCVImageView(parent, name, args)
    {
      try {
        const char* filename = (const char*)args.get(XmNimageFileName);
        int imageLoadingFlag = args.get(XmNimageLoadingFlag);

        originalImage = readImage(filename, imageLoadingFlag);
        
      } catch (SOL::Exception ex) {
        caught(ex);
      }
    }
    
    ~OriginalImageView()
    {
    }
    
    void loadImage(const char* filename, int imageLoadingFlag=CV_LOAD_IMAGE_COLOR)
    {
      try {
        originalImage = readImage(filename, imageLoadingFlag);
        refresh();
      } catch (Exception& ex) {
        caught(ex);
      }
    }
  };
  
  class DetectedImageView :public OpenCVImageView {
  private:
    cv::Mat originalImage;
    cv::Mat edgePreserved;
    cv::Mat grayImage;
    cv::Mat cannyEdgedImage;    
    cv::Mat detectedImage;


    cv::Mat& getMat()
    {
      return detectedImage;
    }
    
    void display()
    {
      show(detectedImage);
    }
    
  public:
    DetectedImageView(View* parent, const char* name, Args& args)
    :OpenCVImageView(parent, name, args)
    {
      try {
        const char* filename = (const char*)args.get(XmNimageFileName);
        int imageLoadingFlag = args.get(XmNimageLoadingFlag);
        loadImage(filename, imageLoadingFlag);       
      } catch (SOL::Exception ex) {
        caught(ex);
      }
    }
    
    ~DetectedImageView()
    {
    }

    void applyEdgePreservedFilter(double sigmaColor, double sigmaSpace, 
                           cv::Mat& image, cv::Mat& filtered )
    {
      try {
        int flag = RECURS_FILTER ; //NORMCONV_FILTER is very slow;
        cv::edgePreservingFilter(
            image, 
            filtered, 
            flag,
            sigmaSpace,        //(double)SIGMA_SPACE, 
            sigmaColor/100.0f); //(double)SIGMA_COLOR/100.0f);
      
      } catch (cv::Exception& ex) {
        ; //Ignore
      }
    }
    
    void loadImage(const char* filename, int imageLoadingFlag=CV_LOAD_IMAGE_COLOR)
    {
      try {
        originalImage = readImage( filename, imageLoadingFlag );
        detectedImage = originalImage.clone(); //create( originalImage.size(), originalImage.type() );

        refresh();
      } catch(Exception& ex) {
        caught(ex);
      }
    }
    
    void detectEdge(double sigmaColor, double sigmaSpace, double threshold1, double threshold2)
    {
      Mat threshold_output;
      vector<vector<Point> > contours;
      vector<Vec4i> hierarchy;
      //1 Apply EdgePreservedFilter to originalImage.
      applyEdgePreservedFilter(sigmaColor, sigmaSpace, originalImage, edgePreserved);
   
      //2 Convert it to grayImage.
      cv::cvtColor(edgePreserved, grayImage, COLOR_BGR2GRAY );
        

      //3 Apply Canny edge detector to the grayImage.
      cv::Canny(grayImage, cannyEdgedImage, threshold1, threshold2);
      
      //4 Find contours from the detectedImage.
      cv::findContours(cannyEdgedImage, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0) );

      vector<vector<Point> > contours_poly( contours.size() );
      vector<Rect> boundRect( contours.size() );
 
      //5 Get approximate polygonal coordinates from the contours, and
      // bounding rectangles from the contours_polygons.
      for( int i = 0; i < contours.size(); i++ ) { 
        cv::approxPolyDP( Mat(contours[i]), contours_poly[i], 3, true );
        boundRect[i] = boundingRect( Mat(contours_poly[i]) );
      }
      detectedImage.release();
      
      detectedImage = originalImage.clone();
      
      //6 Find minimum x and y, and maximum w and h from the boundRect array.
      int width  = detectedImage.size().width;
      int height = detectedImage.size().height;
      int x = width, y = height, w = 0, h = 0;
      int sx = width, sy = height, ex = 0, ey = 0;
      //
      for( int i = 0; i< contours.size(); i++ )  {
        int xx = boundRect[i].x;
        int yy = boundRect[i].y;
        int ww = boundRect[i].width;
        int hh = boundRect[i].height;
        int area = ww * hh;
        int MIN_AREA = 10;
        int MIN_X    = 10;
        int MIN_Y    = 10;
       
        if (area > MIN_AREA) {
          if (xx < sx && xx > MIN_X)
            sx = xx;
          if (yy < sy && yy > MIN_Y)
            sy = yy;
          
          if (xx > ex) 
            ex = xx;
          if (yy > ey)
            ey = yy;
        }
      }
      
      int endx = x + w;
      if (endx >= width)
       endx = width;
      int endy = y + h;
      if (endy >= height)
       endy = height;
      int eex = sx + ex;
      int eey = sy + ey;
      if (eex > width)
        eex = width - 4;
      if (eey > height)
        eey = height- 4;
      //7 Draw a rectangle having a start Point(sx, sy) and end Point(ex, ey) on the contourImage.
      cv::rectangle(detectedImage, Point(sx, sy), Point(ex, ey), CV_RGB(255, 0, 0), 2, 8, 0 );

      refresh();
    }
  };
  //Inner class ends.
  ////////////////////////////////////////////////////////////////////////////////////////
  
  
  StringT<char>                imageFile;
  StringT<char>                savedImageFile;
  
  SmartPtr<OriginalImageView>  originalImage;
  SmartPtr<DetectedImageView>  detectedImage;
  
  static const int             SIGMA_COLOR_MAX =200;
  int                          sigmaColor;
  SmartPtr<LabeledTrackBar>    sigmaColorTrackBar;
  
  static const int             SIGMA_SPACE_MAX=100;
  int                          sigmaSpace;
  SmartPtr<LabeledTrackBar>    sigmaSpaceTrackBar;
  
  static const int             THRESHOLD1_MAX = 300;
  int                          threshold1;
  SmartPtr<LabeledTrackBar>    threshold1TrackBar;
  
  static const int             THRESHOLD2_MAX = 300;
  int                          threshold2;
  SmartPtr<LabeledTrackBar>    threshold2TrackBar;
  
  FileDialog                   filedlg;
  
  void updateCaption()
  {
    char caption[MAX_PATH];
    sprintf_s(caption, CountOf(caption), "%s - %s", 
          (const char*)imageFile, 
          getAppName());
    setText(caption);
  }

  //Horizontal Scroll event by threshold1TrackBar and threshold2TrackBar.
  void commonTrackBarCallback(Action& action)
  {
    sigmaColor = sigmaColorTrackBar -> getPosition();
    sigmaSpace = sigmaSpaceTrackBar -> getPosition();
    
    threshold1 = threshold1TrackBar->getPosition();
    threshold2 = threshold2TrackBar->getPosition();
    
    detectedImage -> detectEdge((double)sigmaColor, (double)sigmaSpace, 
                        (double)threshold1, (double)threshold2);
  }


  void resize(int w, int h)
  {
    if (originalImage && detectedImage &&
          sigmaColorTrackBar && sigmaSpaceTrackBar &&
          threshold1TrackBar && threshold2TrackBar) {
      originalImage      -> reshape(2,            2,  (w-170)/2-1,    h-4);
      detectedImage      -> reshape((w-160)/2+1,  2,  (w-170)/2-1,    h-4);
            
      sigmaColorTrackBar -> reshape(w-165 + 10,  10,      155, 70);
      sigmaSpaceTrackBar -> reshape(w-165 + 10,  10+70,   155, 70);
      threshold1TrackBar -> reshape(w-165 + 10,  10+70*2, 155, 70);
      threshold2TrackBar -> reshape(w-165 + 10,  10+70*3, 155, 70);
    }
  }

  void openFile(const char* filename)
  {
    try {
      originalImage  -> loadImage(filename);
      detectedImage  -> loadImage(filename);
      const char* fname = strrchr(filename, '\\');
      if (fname) {
        fname++;
      }
      imageFile = fname;
      
      updateCaption();
 
    
      detectedImage -> detectEdge((double)sigmaColor, (double)sigmaSpace, 
                        (double)threshold1, (double)threshold2);
      
      savedImageFile = "";
      
    } catch (Exception& ex) {
      caught(ex);
    }
  }
  
  void dropFiles(Action& action)
  {
    char fileName[MAX_PATH] = { 0 };
    DropFiles drop((HDROP)action.getWParam());
    //fileName[0] = ZERO;
    int num = drop.queryFile(0, fileName, CountOf(fileName));
    if(num > 0) {
      if (filedlg.isImageFileName(fileName)) {
        openFile(fileName);
        bringUp();
      } else {        
        bringUp(); //Activate and raise this view
        showErrorDialog("Invalid image filename", fileName,  MB_OK);
      }
    }    
  }

  void confirm(Action& action)
  {
    int rc = MessageBox(NULL, "Are you sure to close this window?", "Confirmation", 
                MB_OKCANCEL|MB_ICONEXCLAMATION);
    if (rc == IDOK) {
      exit(action);
    }
  }
  
public:
  MainView(OpenCVApplication& applet, const char* name, Args& args)
  :OpenCVApplicationView(applet, name, args)
  {
    imageFile = "..\\images\\CatImage.png";
    int imageLoadingFlag = CV_LOAD_IMAGE_COLOR;
    try {
      Args ar;
      ar.set(XmNimageFileName, imageFile);
      ar.set(XmNimageLoadingFlag, imageLoadingFlag);
      originalImage = new OriginalImageView(this, "cvwindow1", ar); 
      originalImage -> addCallback(XmNdropCallback, this,
        (Callback)&MainView::dropFiles, NULL);
      
      ar.reset();
      ar.set(XmNimageFileName, imageFile);
      ar.set(XmNimageLoadingFlag, imageLoadingFlag);
      detectedImage = new DetectedImageView(this, "cvwindow2", ar); 

      sigmaColor = 120;
      sigmaSpace = 40;
      
      ar.reset();
      ar.set(XmNminimum, 0);
      ar.set(XmNmaximum, SIGMA_COLOR_MAX);
      ar.set(XmNposition, sigmaColor);
      ar.set(XmNdisplayOddValue,   false);
      ar.set(XmNdisplayFloatValue, false);
      sigmaColorTrackBar = new LabeledTrackBar(this, "Edge SigmaColor", ar);
      sigmaColorTrackBar -> addCallback(XmNtrackBarScrollCallback, this,
        (Callback)&MainView::commonTrackBarCallback, NULL);
      
      ar.reset();
      ar.set(XmNminimum, 0);
      ar.set(XmNmaximum, SIGMA_SPACE_MAX);
      ar.set(XmNposition, sigmaSpace);
      ar.set(XmNdisplayFloatValue, true);
      ar.set(XmNdisplayOddValue,   false);
      sigmaSpaceTrackBar = new LabeledTrackBar(this, "Edge SigmaSpace", ar);
      sigmaSpaceTrackBar -> addCallback(XmNtrackBarScrollCallback, this,
        (Callback)&MainView::commonTrackBarCallback, NULL);

      threshold1 =  43;
      threshold2 =  110;
        
      ar.reset();
      ar.set(XmNminimum, 0);
      ar.set(XmNmaximum, THRESHOLD1_MAX);
      ar.set(XmNposition, threshold1);
      ar.set(XmNdisplayOddValue, false);
      threshold1TrackBar = new LabeledTrackBar(this, "Canny Threshold1", ar);
      
      threshold1TrackBar -> addCallback(XmNtrackBarScrollCallback, this,
        (Callback)&MainView::commonTrackBarCallback, NULL);
            
      ar.reset();
      ar.set(XmNminimum, 0);
      ar.set(XmNmaximum, THRESHOLD2_MAX);
      ar.set(XmNposition, threshold2);
      ar.set(XmNdisplayOddValue, false);
      threshold2TrackBar = new LabeledTrackBar(this, "Canny Threshold2", ar);

      threshold2TrackBar -> addCallback(XmNtrackBarScrollCallback, this,
        (Callback)&MainView::commonTrackBarCallback, NULL);
            
      addCallback(XmNmenuCallback, IDM_EXIT, this,
          (Callback)&MainView::confirm, NULL);
      
    
      detectedImage -> detectEdge((double)sigmaColor, (double)sigmaSpace, 
                        (double)threshold1, (double)threshold2);
      
      ar.reset();
      ar.set(XmNfilter, FileDialog::getImageFilesFilter());
      filedlg.create(this, "OpenFile", ar);

      updateCaption();
      
    } catch (Exception& ex) {
      caught(ex);
    }
  }

  ~MainView()
  {
  }
  
  void open(Action& action)
  {
    Args ar;
    
    char dir[MAX_PATH] = { 0 };
    if (restoreFileFolder(dir, CountOf(dir))) {
      ar.set(XmNdirectory, dir);
      filedlg.setValues(ar);
    }
    
    try {    
      if(filedlg.open()) {
    
        const char* filename = filedlg.getFileName();
        saveFileFolder(filename);
        
        openFile(filename);
        
      }
    } catch (Exception& ex) {
      caught(ex);
    }
  }

  //2017/12/01
  void save(Action& action)
  {
    try {
      if (!savedImageFile.isEmpty()) {
        //Write detected image into the filename.
        detectedImage->writeImage(savedImageFile);
      } else {
        saveAs(action);
      }
    } catch (Exception& ex) {
      caught(ex);
    }
  }
  
  //2017/12/01
  void saveAs(Action& action)
  {
    Args ar;
    
    char dir[MAX_PATH] = { 0 };
    if (restoreFileFolder(dir, CountOf(dir))) {
      ar.set(XmNdirectory, dir);
      filedlg.setValues(ar);
    }
    
    try {    
      if(filedlg.save()) {
    
        const char* filename = filedlg.getFileName();
        saveFileFolder(filename);
        
        //Write detected image into the filename.
        detectedImage->writeImage(filename);
        savedImageFile = filename;
      }
    } catch (Exception& ex) {
      caught(ex);
    }
  }  
};
}

//
void main(int argc, char** argv) 
{
  try {
    ModuleFileName module(argv[0]);
    
    const char*  name = module.getAppName();
        
    OpenCVApplication applet(name, argc, argv);

    Args args;
    args.set(XmNwidth,  860);
    args.set(XmNheight, 360);
    MainView view(applet, name, args);
    view.realize();

    applet.run();
    
  } catch (SOL::Exception& ex) {
    caught(ex);
  }
}


Last modified: 15 Apr. 2018

Copyright (c) 2018 Antillia.com ALL RIGHTS RESERVED.