Es++ Sample: ShapeMatcher |
/****************************************************************************** * * Copyright (c) 2017 Antillia.com TOSHIYUKI ARAI. ALL RIGHTS RESERVED. * * Redistribution and use in binarized and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of binarized code must retain the above copyright * notice, this list of conditions, and the following disclaimer. * * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * * ShapeMatcher.cpp * *****************************************************************************/ #include <es++/Pair.h> #include <es++/gtkmm-3.0/Application.h> #include <es++/gtkmm-3.0/FileOpenDialog.h> #include <es++/gtkmm-3.0/Label.h> #include <es++/gtkmm-3.0/LabeledComboBox.h> #include <es++/gtkmm-3.0/LabeledTrackBar.h> #include <es++/gtkmm-3.0/PushButton.h> #include <es++/opencv-4.0/OpenCVMainView.h> #include <es++/opencv-4.0/OpenCVScaleComboBox.h> #include <es++/opencv-4.0/OpenCVScrolledImageView.h> using namespace Gtk; namespace Es { class MainView :public Es::OpenCVMainView { private: ///////////////////////////////////////////////////// //Inner classes start. class BinarizedImageView :public Es::OpenCVScrolledImageView { private: cv::Mat gray_image; cv::Mat binarized_image; public: BinarizedImageView() { } void loadImage(const std::string& filename, int imageLoadingFlag= CV_LOAD_IMAGE_COLOR, int scaling_ratio=100) { OpenCVScrolledImageView::loadImage(filename, imageLoadingFlag, scaling_ratio); cv::Mat& original_image = getOriginalImage(); cv::cvtColor(original_image, gray_image, COLOR_BGR2GRAY); binarized_image = gray_image.clone(); scaleImage(binarized_image, scaling_ratio); } cv::Mat& getBinarizedImage() { return binarized_image; } void binarize(int threshold_type, int threshold_value, int scaling_ratio) { try { cv::threshold( gray_image, binarized_image, threshold_value, THRESHOLD_VALUE_MAX, threshold_type ); } catch (cv::Exception& ex) { //Sometimes we get an exception; I don't know the reason why it happens. } scaleImage(binarized_image, scaling_ratio); } void rescale(int scaling_ratio) { OpenCVScrolledImageView::scaleImage(binarized_image, scaling_ratio); } }; class MatchedImageView :public Es::OpenCVScrolledImageView { private: cv::Mat matched_image; public: MatchedImageView() { } void loadImage(const std::string& filename, int imageLoadingFlag= CV_LOAD_IMAGE_COLOR, int scaling_ratio=100) { OpenCVScrolledImageView::loadImage(filename, imageLoadingFlag, scaling_ratio); cv::Mat& original_image = getOriginalImage(); matched_image = original_image.clone(); scaleImage(matched_image, scaling_ratio); } void rescale(int scaling_ratio) { OpenCVScrolledImageView::scaleImage(matched_image, scaling_ratio); } void setMatched(cv::Mat& mat, int scaling_ratio) { matched_image = mat.clone(); scaleImage(matched_image, scaling_ratio); } }; //Inner classes end. ///////////////////////////////////////////////////// // private: static const int IMAGES_COUNT = 2; static const int FIRST = 0; static const int SECOND = 1; std::string filenames[IMAGES_COUNT]; Es::Label filepath; Es::HorizontalLayout horiz_layout; Es::VerticalLayout view_layout; Es::HorizontalLayout binarized_view_layout; BinarizedImageView binarized_images[IMAGES_COUNT]; MatchedImageView matched_image; Es::VerticalLayout control_pane; Es::OpenCVScaleComboBox combobox; Es::LabeledComboBox threshold_type_combobox; Es::LabeledTrackBar threshold_value_trackbar; Es::LabeledTrackBar minimum_size_trackbar; Es::LabeledTrackBar maximum_size_trackbar; Es::PushButton match_button; int threshold_type_index; int threshold_value; int minimum_size; int maximum_size; static const int FILEPATH_HEIGHT = 30; static const int CONTROLPANE_WIDTH = 200; int scaling_ratio; static const int loading_flag = IMREAD_COLOR; Es::FileOpenDialog file_dialog; static const int THRESHOLD_VALUE_MAX = 255; public: ////////////////////////////////////////////// //Constructor // MainView(Es::Application& applet, std::string& title, Es::Args& args) :OpenCVMainView(applet, title, //We don't use the defaultFilePulldown menu. args.set(XmNuseDefaultFilePulldown, false) ) ,scaling_ratio(100) ,binarized_view_layout(0) //SPACING ,file_dialog(*this, Es::FileOpenDialog::IMAGE_FILES) { //Define File pulldown menu with multiple [Open File] pulldown menu items. Es::SigcMenuCallback file_menu_callbacks[] = { {"New", sigc::mem_fun(*this, &MainView::file_new)}, {"Open File1", sigc::mem_fun(*this, &MainView::file_open1)}, {"Open File2", sigc::mem_fun(*this, &MainView::file_open2)}, {"Save", sigc::mem_fun(*this, &MainView::file_save)}, {"Save_As", sigc::mem_fun(*this, &MainView::file_save_as)}, {"Quit", sigc::mem_fun(*this, &MainView::file_quit)}, }; file_pulldown_append( file_menu_callbacks, CountOf(file_menu_callbacks)); int w = (int)args.get(XmNwidth); int h = (int)args.get(XmNheight); int ww = (w - CONTROLPANE_WIDTH)/2; int hh = h/2; int ratio = (int)args.get(XmNscalingRatio); scaling_ratio = OpenCVImageView::validateScale(ratio); Es::MainLayout& main_layout = get_main_layout(); filepath.set_size_request(w, FILEPATH_HEIGHT); filepath.set_alignment(Gtk::ALIGN_START); main_layout.pack_start(filepath, FALSE,FALSE, 0); main_layout.pack_start(horiz_layout, FALSE, FALSE, 0); binarized_images[FIRST].set_size_request(ww, h/2); binarized_images[SECOND].set_size_request(ww, h/2); matched_image.set_size_request(w, h/2); control_pane.set_spacing(10); control_pane.set_size_request(CONTROLPANE_WIDTH, h); main_layout.pack_start(filepath, FALSE, FALSE, 0); main_layout.pack_start(horiz_layout); horiz_layout.pack_start(view_layout); view_layout.pack_start(binarized_view_layout); binarized_view_layout.pack_start(binarized_images[FIRST]); binarized_view_layout.pack_start(binarized_images[SECOND]); view_layout.pack_start(matched_image); horiz_layout.pack_start(control_pane); control_pane.pack_start(combobox, Gtk::PACK_SHRINK ); match_button.set_label("Match"); control_pane.pack_start(threshold_type_combobox, Gtk::PACK_SHRINK); control_pane.pack_start(threshold_value_trackbar, Gtk::PACK_SHRINK); control_pane.pack_start(minimum_size_trackbar, Gtk::PACK_SHRINK); control_pane.pack_start(maximum_size_trackbar, Gtk::PACK_SHRINK); control_pane.pack_start(match_button, Gtk::PACK_SHRINK); combobox.set_selection(scaling_ratio); combobox.set_changed_callback(sigc::mem_fun(*this, &MainView::scale_changed) ); match_button.set_label("Match"); match_button.set_clicked_callback(sigc::mem_fun(*this, &MainView::match_button_clicked) ); //Create a threshold type comboBox in controlPane. const char* types[] = { "Binary", "Binary Inverted", "Threshold Truncated", "Threshold to Zero", "Threshold to Zero Inverted", //"Mask", "Otsu", }; threshold_type_index = 0; threshold_type_combobox.set_label("ThresholdType"); threshold_type_combobox.append_items(types, CountOf(types)); threshold_type_combobox.set_active_text( types[threshold_type_index]);//Binary threshold_type_combobox.set_changed_callback(sigc::mem_fun(*this, &MainView::threshold_type_changed) ); threshold_value_trackbar.set_label("ThresholdValue:[0,255]"); threshold_value = 60; threshold_value_trackbar.configure_adjuster(threshold_value, 0, 255); threshold_value_trackbar.set_value_changed_callback( sigc::mem_fun(*this, &MainView::threshold_value_changed) ); minimum_size = 11; minimum_size_trackbar.set_label("MatchMinSize:[10,100]"); minimum_size_trackbar.configure_adjuster(minimum_size, 0, 100); minimum_size_trackbar.set_value_changed_callback( sigc::mem_fun(*this, &MainView::minimum_size_changed) ); maximum_size = 101; maximum_size_trackbar.set_label("MatchMaxSize:[100,400]"); maximum_size_trackbar.configure_adjuster(maximum_size, 100, 400); maximum_size_trackbar.set_value_changed_callback( sigc::mem_fun(*this, &MainView::maximum_size_changed) ); filenames[FIRST] = "../../images/CatImage.png"; filenames[SECOND] = "../../images/CatFace.png"; for (int i = 0; i<IMAGES_COUNT; i++) { binarized_images[i].loadImage(filenames[i], loading_flag, scaling_ratio); binarized_images[i].binarize( getThresholdType(threshold_type_index), threshold_value, scaling_ratio ); } //binarized_images[SECOND].loadImage(filenames[SECOND], loading_flag, scaling_ratio); matched_image.loadImage(filenames[FIRST], loading_flag, scaling_ratio); /* binarized_images[FIRST].binarize( getThresholdType(threshold_type_index), threshold_value, scaling_ratio ); binarized_images[SECOND].binarize( getThresholdType(threshold_type_index), threshold_value, scaling_ratio ); */ update_filepath(); show_all(); } void scale_changed() { scaling_ratio = combobox.get_selection(); printf("scale_changed %d\n", scaling_ratio); binarized_images[FIRST].rescale(scaling_ratio); binarized_images[SECOND].rescale(scaling_ratio); matched_image.rescale(scaling_ratio); } void update_filepath() { std::string space = " "; filepath.set_label(filenames[FIRST] + space + filenames[SECOND]); } static int getThresholdType(int index) { int n = 0; //We don't include the THRESH_OTSU type Pair<int, int> types[] = { {n++, THRESH_BINARY }, {n++, THRESH_BINARY_INV }, {n++, THRESH_TRUNC }, {n++, THRESH_TOZERO }, {n++, THRESH_TOZERO_INV }, {n++, THRESH_OTSU }, }; int type = THRESH_BINARY; if (index >= 0 && index <n) { type = types[index].second; } return type; } void threshold_type_changed() { int prev_threshold_type = threshold_type_index; threshold_type_index = threshold_type_combobox.get_active_row_number(); printf("threshold_type_index %d\n", threshold_type_index); if (threshold_type_index != prev_threshold_type) { printf("Detector updated call match\n"); match_button_clicked(); } } void threshold_value_changed() { threshold_value = threshold_value_trackbar.get_value(); match_button_clicked(); } void minimum_size_changed() { minimum_size = minimum_size_trackbar.get_value(); match_button_clicked(); } void maximum_size_changed() { maximum_size = maximum_size_trackbar.get_value(); match_button_clicked(); } //Matching operation. void match_button_clicked() { //Detect connectedCommponentsWithStats on destImage Mat labels, stats, centroids; cv::Mat& originalImage = binarized_images[FIRST ].getOriginalImage(); cv::Mat& srcBin = binarized_images[FIRST ].getBinarizedImage(); cv::Mat& tmpBin = binarized_images[SECOND].getBinarizedImage(); cv::connectedComponentsWithStats(srcBin, labels, stats, centroids); int tw = tmpBin.rows; int th = tmpBin.cols; cv::Mat destImage = originalImage.clone(); const double MATCHING_THRESHOLD = 0.005; cv::Rect minimum(0, 0, 0, 0); double MIN_SIMILARITY = 1.0; bool found = false; for(int i=0; i < stats.rows; i++) { int x = stats.at<int>(Point(0, i)); int y = stats.at<int>(Point(1, i)); int w = stats.at<int>(Point(2, i)); int h = stats.at<int>(Point(3, i)); cv::Rect rect(x, y, w, h); cv::Mat rectOfInterest = srcBin(rect); double similarity = cv::matchShapes(tmpBin, rectOfInterest, CV_CONTOURS_MATCH_I1, 0); if ( (w >= minimum_size || h >= minimum_size ) && (w <= maximum_size || h <= maximum_size )) { if (similarity <= MIN_SIMILARITY) { MIN_SIMILARITY = similarity; minimum = rect; printf("matching similarity=%.04f x=%d y=%d w=%d h=%d\n", similarity, x, y, w, h); found = true; } } } if (found) { printf("found \n"); cv::rectangle(destImage, minimum, CV_RGB(255, 0, 0), 3); } matched_image.setMatched(destImage, scaling_ratio); } void file_open1() { int rc = file_dialog.popup(); if (rc == Gtk::RESPONSE_OK) { filenames[FIRST] = file_dialog.get_filename(); update_filepath(); binarized_images[FIRST].loadImage(filenames[FIRST], loading_flag, scaling_ratio); binarized_images[FIRST].binarize( getThresholdType(threshold_type_index), threshold_value, scaling_ratio ); } } void file_open2() { int rc = file_dialog.popup(); if (rc == Gtk::RESPONSE_OK) { filenames[SECOND] = file_dialog.get_filename(); update_filepath(); binarized_images[SECOND].loadImage(filenames[SECOND], loading_flag, scaling_ratio); binarized_images[SECOND].binarize( getThresholdType(threshold_type_index), threshold_value, scaling_ratio ); } } }; } int main(int argc, char** argv) { Es::Environment env; try { Es::Application applet(argc, argv); std::string title(argv[0]); Es::Args args; args.set(XmNx, 20); args.set(XmNy, 20); args.set(XmNwidth, 900); args.set(XmNheight,460); args.set(XmNscalingRatio,60); Es::MainView mainv(applet, title, args); mainv.show(); applet.run(mainv); } catch (Es::Exception& ex) { caught(ex); } catch (...) { printf("Get exception \n"); } return 0; }