summaryrefslogtreecommitdiffstats
path: root/ml/dlib/tools/imglab/src/metadata_editor.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'ml/dlib/tools/imglab/src/metadata_editor.cpp')
-rw-r--r--ml/dlib/tools/imglab/src/metadata_editor.cpp671
1 files changed, 671 insertions, 0 deletions
diff --git a/ml/dlib/tools/imglab/src/metadata_editor.cpp b/ml/dlib/tools/imglab/src/metadata_editor.cpp
new file mode 100644
index 000000000..76177e893
--- /dev/null
+++ b/ml/dlib/tools/imglab/src/metadata_editor.cpp
@@ -0,0 +1,671 @@
+// Copyright (C) 2011 Davis E. King (davis@dlib.net)
+// License: Boost Software License See LICENSE.txt for the full license.
+
+#include "metadata_editor.h"
+#include <dlib/array.h>
+#include <dlib/queue.h>
+#include <dlib/static_set.h>
+#include <dlib/misc_api.h>
+#include <dlib/image_io.h>
+#include <dlib/array2d.h>
+#include <dlib/pixel.h>
+#include <dlib/image_transforms.h>
+#include <dlib/image_processing.h>
+#include <sstream>
+#include <ctime>
+
+using namespace std;
+using namespace dlib;
+
+extern const char* VERSION;
+
+// ----------------------------------------------------------------------------------------
+
+metadata_editor::
+metadata_editor(
+ const std::string& filename_
+) :
+ mbar(*this),
+ lb_images(*this),
+ image_pos(0),
+ display(*this),
+ overlay_label_name(*this),
+ overlay_label(*this),
+ keyboard_jump_pos(0),
+ last_keyboard_jump_pos_update(0)
+{
+ file metadata_file(filename_);
+ filename = metadata_file.full_name();
+ // Make our current directory be the one that contains the metadata file. We
+ // do this because that file might contain relative paths to the image files
+ // we are supposed to be loading.
+ set_current_dir(get_parent_directory(metadata_file).full_name());
+
+ load_image_dataset_metadata(metadata, filename);
+
+ dlib::array<std::string>::expand_1a files;
+ files.resize(metadata.images.size());
+ for (unsigned long i = 0; i < metadata.images.size(); ++i)
+ {
+ files[i] = metadata.images[i].filename;
+ }
+ lb_images.load(files);
+ lb_images.enable_multiple_select();
+
+ lb_images.set_click_handler(*this, &metadata_editor::on_lb_images_clicked);
+
+ overlay_label_name.set_text("Next Label: ");
+ overlay_label.set_width(200);
+
+ display.set_image_clicked_handler(*this, &metadata_editor::on_image_clicked);
+ display.set_overlay_rects_changed_handler(*this, &metadata_editor::on_overlay_rects_changed);
+ display.set_overlay_rect_selected_handler(*this, &metadata_editor::on_overlay_rect_selected);
+ overlay_label.set_text_modified_handler(*this, &metadata_editor::on_overlay_label_changed);
+
+ mbar.set_number_of_menus(2);
+ mbar.set_menu_name(0,"File",'F');
+ mbar.set_menu_name(1,"Help",'H');
+
+
+ mbar.menu(0).add_menu_item(menu_item_text("Save",*this,&metadata_editor::file_save,'S'));
+ mbar.menu(0).add_menu_item(menu_item_text("Save As",*this,&metadata_editor::file_save_as,'A'));
+ mbar.menu(0).add_menu_item(menu_item_separator());
+ mbar.menu(0).add_menu_item(menu_item_text("Remove Selected Images",*this,&metadata_editor::remove_selected_images,'R'));
+ mbar.menu(0).add_menu_item(menu_item_separator());
+ mbar.menu(0).add_menu_item(menu_item_text("Exit",static_cast<base_window&>(*this),&drawable_window::close_window,'x'));
+
+ mbar.menu(1).add_menu_item(menu_item_text("About",*this,&metadata_editor::display_about,'A'));
+
+ // set the size of this window.
+ on_window_resized();
+ load_image_and_set_size(0);
+ on_window_resized();
+ if (image_pos < lb_images.size() )
+ lb_images.select(image_pos);
+
+ // make sure the window is centered on the screen.
+ unsigned long width, height;
+ get_size(width, height);
+ unsigned long screen_width, screen_height;
+ get_display_size(screen_width, screen_height);
+ set_pos((screen_width-width)/2, (screen_height-height)/2);
+
+ show();
+}
+
+// ----------------------------------------------------------------------------------------
+
+metadata_editor::
+~metadata_editor(
+)
+{
+ close_window();
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+add_labelable_part_name (
+ const std::string& name
+)
+{
+ display.add_labelable_part_name(name);
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+file_save()
+{
+ save_metadata_to_file(filename);
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+save_metadata_to_file (
+ const std::string& file
+)
+{
+ try
+ {
+ save_image_dataset_metadata(metadata, file);
+ }
+ catch (dlib::error& e)
+ {
+ message_box("Error saving file", e.what());
+ }
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+file_save_as()
+{
+ save_file_box(*this, &metadata_editor::save_metadata_to_file);
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+remove_selected_images()
+{
+ dlib::queue<unsigned long>::kernel_1a list;
+ lb_images.get_selected(list);
+ list.reset();
+ unsigned long min_idx = lb_images.size();
+ while (list.move_next())
+ {
+ lb_images.unselect(list.element());
+ min_idx = std::min(min_idx, list.element());
+ }
+
+
+ // remove all the selected items from metadata.images
+ dlib::static_set<unsigned long>::kernel_1a to_remove;
+ to_remove.load(list);
+ std::vector<dlib::image_dataset_metadata::image> images;
+ for (unsigned long i = 0; i < metadata.images.size(); ++i)
+ {
+ if (to_remove.is_member(i) == false)
+ {
+ images.push_back(metadata.images[i]);
+ }
+ }
+ images.swap(metadata.images);
+
+
+ // reload metadata into lb_images
+ dlib::array<std::string>::expand_1a files;
+ files.resize(metadata.images.size());
+ for (unsigned long i = 0; i < metadata.images.size(); ++i)
+ {
+ files[i] = metadata.images[i].filename;
+ }
+ lb_images.load(files);
+
+
+ if (min_idx != 0)
+ min_idx--;
+ select_image(min_idx);
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+on_window_resized(
+)
+{
+ drawable_window::on_window_resized();
+
+ unsigned long width, height;
+ get_size(width, height);
+
+ lb_images.set_pos(0,mbar.bottom()+1);
+ lb_images.set_size(180, height - mbar.height());
+
+ overlay_label_name.set_pos(lb_images.right()+10, mbar.bottom() + (overlay_label.height()-overlay_label_name.height())/2+1);
+ overlay_label.set_pos(overlay_label_name.right(), mbar.bottom()+1);
+ display.set_pos(lb_images.right(), overlay_label.bottom()+3);
+
+ display.set_size(width - display.left(), height - display.top());
+}
+
+// ----------------------------------------------------------------------------------------
+
+void propagate_boxes(
+ dlib::image_dataset_metadata::dataset& data,
+ unsigned long prev,
+ unsigned long next
+)
+{
+ if (prev == next || next >= data.images.size())
+ return;
+
+ array2d<rgb_pixel> img1, img2;
+ dlib::load_image(img1, data.images[prev].filename);
+ dlib::load_image(img2, data.images[next].filename);
+ for (unsigned long i = 0; i < data.images[prev].boxes.size(); ++i)
+ {
+ correlation_tracker tracker;
+ tracker.start_track(img1, data.images[prev].boxes[i].rect);
+ tracker.update(img2);
+ dlib::image_dataset_metadata::box box = data.images[prev].boxes[i];
+ box.rect = tracker.get_position();
+ data.images[next].boxes.push_back(box);
+ }
+}
+
+// ----------------------------------------------------------------------------------------
+
+void propagate_labels(
+ const std::string& label,
+ dlib::image_dataset_metadata::dataset& data,
+ unsigned long prev,
+ unsigned long next
+)
+{
+ if (prev == next || next >= data.images.size())
+ return;
+
+
+ for (unsigned long i = 0; i < data.images[prev].boxes.size(); ++i)
+ {
+ if (data.images[prev].boxes[i].label != label)
+ continue;
+
+ // figure out which box in the next image matches the current one the best
+ const rectangle cur = data.images[prev].boxes[i].rect;
+ double best_overlap = 0;
+ unsigned long best_idx = 0;
+ for (unsigned long j = 0; j < data.images[next].boxes.size(); ++j)
+ {
+ const rectangle next_box = data.images[next].boxes[j].rect;
+ const double overlap = cur.intersect(next_box).area()/(double)(cur+next_box).area();
+ if (overlap > best_overlap)
+ {
+ best_overlap = overlap;
+ best_idx = j;
+ }
+ }
+
+ // If we found a matching rectangle in the next image and the best match doesn't
+ // already have a label.
+ if (best_overlap > 0.5 && data.images[next].boxes[best_idx].label == "")
+ {
+ data.images[next].boxes[best_idx].label = label;
+ }
+ }
+
+}
+
+// ----------------------------------------------------------------------------------------
+
+bool has_label_or_all_boxes_labeled (
+ const std::string& label,
+ const dlib::image_dataset_metadata::image& img
+)
+{
+ if (label.size() == 0)
+ return true;
+
+ bool all_boxes_labeled = true;
+ for (unsigned long i = 0; i < img.boxes.size(); ++i)
+ {
+ if (img.boxes[i].label == label)
+ return true;
+ if (img.boxes[i].label.size() == 0)
+ all_boxes_labeled = false;
+ }
+
+ return all_boxes_labeled;
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+on_keydown (
+ unsigned long key,
+ bool is_printable,
+ unsigned long state
+)
+{
+ drawable_window::on_keydown(key, is_printable, state);
+
+ if (is_printable)
+ {
+ if (key == '\t')
+ {
+ overlay_label.give_input_focus();
+ overlay_label.select_all_text();
+ }
+
+ // If the user types a number then jump to that image.
+ if ('0' <= key && key <= '9' && metadata.images.size() != 0 && !overlay_label.has_input_focus())
+ {
+ time_t curtime = time(0);
+ // If it's been a while since the user typed numbers then forget the last jump
+ // position and start accumulating numbers over again.
+ if (curtime-last_keyboard_jump_pos_update >= 2)
+ keyboard_jump_pos = 0;
+ last_keyboard_jump_pos_update = curtime;
+
+ keyboard_jump_pos *= 10;
+ keyboard_jump_pos += key-'0';
+ if (keyboard_jump_pos >= metadata.images.size())
+ keyboard_jump_pos = metadata.images.size()-1;
+
+ image_pos = keyboard_jump_pos;
+ select_image(image_pos);
+ }
+ else
+ {
+ last_keyboard_jump_pos_update = 0;
+ }
+
+ if (key == 'd' && (state&base_window::KBD_MOD_ALT))
+ {
+ remove_selected_images();
+ }
+
+ if (key == 'e' && !overlay_label.has_input_focus())
+ {
+ display_equialized_image = !display_equialized_image;
+ select_image(image_pos);
+ }
+
+ // Make 'w' and 's' act like KEY_UP and KEY_DOWN
+ if ((key == 'w' || key == 'W') && !overlay_label.has_input_focus())
+ {
+ key = base_window::KEY_UP;
+ }
+ else if ((key == 's' || key == 'S') && !overlay_label.has_input_focus())
+ {
+ key = base_window::KEY_DOWN;
+ }
+ else
+ {
+ return;
+ }
+ }
+
+ if (key == base_window::KEY_UP)
+ {
+ if ((state&KBD_MOD_CONTROL) && (state&KBD_MOD_SHIFT))
+ {
+ // Don't do anything if there are no boxes in the current image.
+ if (metadata.images[image_pos].boxes.size() == 0)
+ return;
+ // Also don't do anything if there *are* boxes in the next image.
+ if (image_pos > 1 && metadata.images[image_pos-1].boxes.size() != 0)
+ return;
+
+ propagate_boxes(metadata, image_pos, image_pos-1);
+ }
+ else if (state&base_window::KBD_MOD_CONTROL)
+ {
+ // If the label we are supposed to propagate doesn't exist in the current image
+ // then don't advance.
+ if (!has_label_or_all_boxes_labeled(display.get_default_overlay_rect_label(),metadata.images[image_pos]))
+ return;
+
+ // if the next image is going to be empty then fast forward to the next one
+ while (image_pos > 1 && metadata.images[image_pos-1].boxes.size() == 0)
+ --image_pos;
+
+ propagate_labels(display.get_default_overlay_rect_label(), metadata, image_pos, image_pos-1);
+ }
+ select_image(image_pos-1);
+ }
+ else if (key == base_window::KEY_DOWN)
+ {
+ if ((state&KBD_MOD_CONTROL) && (state&KBD_MOD_SHIFT))
+ {
+ // Don't do anything if there are no boxes in the current image.
+ if (metadata.images[image_pos].boxes.size() == 0)
+ return;
+ // Also don't do anything if there *are* boxes in the next image.
+ if (image_pos+1 < metadata.images.size() && metadata.images[image_pos+1].boxes.size() != 0)
+ return;
+
+ propagate_boxes(metadata, image_pos, image_pos+1);
+ }
+ else if (state&base_window::KBD_MOD_CONTROL)
+ {
+ // If the label we are supposed to propagate doesn't exist in the current image
+ // then don't advance.
+ if (!has_label_or_all_boxes_labeled(display.get_default_overlay_rect_label(),metadata.images[image_pos]))
+ return;
+
+ // if the next image is going to be empty then fast forward to the next one
+ while (image_pos+1 < metadata.images.size() && metadata.images[image_pos+1].boxes.size() == 0)
+ ++image_pos;
+
+ propagate_labels(display.get_default_overlay_rect_label(), metadata, image_pos, image_pos+1);
+ }
+ select_image(image_pos+1);
+ }
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+select_image(
+ unsigned long idx
+)
+{
+ if (idx < lb_images.size())
+ {
+ // unselect all currently selected images
+ dlib::queue<unsigned long>::kernel_1a list;
+ lb_images.get_selected(list);
+ list.reset();
+ while (list.move_next())
+ {
+ lb_images.unselect(list.element());
+ }
+
+
+ lb_images.select(idx);
+ load_image(idx);
+ }
+ else if (lb_images.size() == 0)
+ {
+ display.clear_overlay();
+ array2d<unsigned char> empty_img;
+ display.set_image(empty_img);
+ }
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+on_lb_images_clicked(
+ unsigned long idx
+)
+{
+ load_image(idx);
+}
+
+// ----------------------------------------------------------------------------------------
+
+std::vector<dlib::image_display::overlay_rect> get_overlays (
+ const dlib::image_dataset_metadata::image& data,
+ color_mapper& string_to_color
+)
+{
+ std::vector<dlib::image_display::overlay_rect> temp(data.boxes.size());
+ for (unsigned long i = 0; i < temp.size(); ++i)
+ {
+ temp[i].rect = data.boxes[i].rect;
+ temp[i].label = data.boxes[i].label;
+ temp[i].parts = data.boxes[i].parts;
+ temp[i].crossed_out = data.boxes[i].ignore;
+ temp[i].color = string_to_color(data.boxes[i].label);
+ }
+ return temp;
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+load_image(
+ unsigned long idx
+)
+{
+ if (idx >= metadata.images.size())
+ return;
+
+ image_pos = idx;
+
+ array2d<rgb_pixel> img;
+ display.clear_overlay();
+ try
+ {
+ dlib::load_image(img, metadata.images[idx].filename);
+ set_title(metadata.name + " #"+cast_to_string(idx)+": " +metadata.images[idx].filename);
+ }
+ catch (exception& e)
+ {
+ message_box("Error loading image", e.what());
+ }
+
+ if (display_equialized_image)
+ equalize_histogram(img);
+ display.set_image(img);
+ display.add_overlay(get_overlays(metadata.images[idx], string_to_color));
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+load_image_and_set_size(
+ unsigned long idx
+)
+{
+ if (idx >= metadata.images.size())
+ return;
+
+ image_pos = idx;
+
+ array2d<rgb_pixel> img;
+ display.clear_overlay();
+ try
+ {
+ dlib::load_image(img, metadata.images[idx].filename);
+ set_title(metadata.name + " #"+cast_to_string(idx)+": " +metadata.images[idx].filename);
+ }
+ catch (exception& e)
+ {
+ message_box("Error loading image", e.what());
+ }
+
+
+ unsigned long screen_width, screen_height;
+ get_display_size(screen_width, screen_height);
+
+
+ unsigned long needed_width = display.left() + img.nc() + 4;
+ unsigned long needed_height = display.top() + img.nr() + 4;
+ if (needed_width < 300) needed_width = 300;
+ if (needed_height < 300) needed_height = 300;
+
+ if (needed_width > 100 + screen_width)
+ needed_width = screen_width - 100;
+ if (needed_height > 100 + screen_height)
+ needed_height = screen_height - 100;
+
+ set_size(needed_width, needed_height);
+
+
+ if (display_equialized_image)
+ equalize_histogram(img);
+ display.set_image(img);
+ display.add_overlay(get_overlays(metadata.images[idx], string_to_color));
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+on_overlay_rects_changed(
+)
+{
+ using namespace dlib::image_dataset_metadata;
+ if (image_pos < metadata.images.size())
+ {
+ const std::vector<image_display::overlay_rect>& rects = display.get_overlay_rects();
+
+ std::vector<box>& boxes = metadata.images[image_pos].boxes;
+
+ boxes.clear();
+ for (unsigned long i = 0; i < rects.size(); ++i)
+ {
+ box temp;
+ temp.label = rects[i].label;
+ temp.rect = rects[i].rect;
+ temp.parts = rects[i].parts;
+ temp.ignore = rects[i].crossed_out;
+ boxes.push_back(temp);
+ }
+ }
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+on_image_clicked(
+ const point& /*p*/, bool /*is_double_click*/, unsigned long /*btn*/
+)
+{
+ display.set_default_overlay_rect_color(string_to_color(trim(overlay_label.text())));
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+on_overlay_label_changed(
+)
+{
+ display.set_default_overlay_rect_label(trim(overlay_label.text()));
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+on_overlay_rect_selected(
+ const image_display::overlay_rect& orect
+)
+{
+ overlay_label.set_text(orect.label);
+ display.set_default_overlay_rect_label(orect.label);
+ display.set_default_overlay_rect_color(string_to_color(orect.label));
+}
+
+// ----------------------------------------------------------------------------------------
+
+void metadata_editor::
+display_about(
+)
+{
+ std::ostringstream sout;
+ sout << wrap_string("Image Labeler v" + string(VERSION) + "." ,0,0) << endl << endl;
+ sout << wrap_string("This program is a tool for labeling images with rectangles. " ,0,0) << endl << endl;
+
+ sout << wrap_string("You can add a new rectangle by holding the shift key, left clicking "
+ "the mouse, and dragging it. New rectangles are given the label from the \"Next Label\" "
+ "field at the top of the application. You can quickly edit the contents of the Next Label field "
+ "by hitting the tab key. Double clicking "
+ "a rectangle selects it and the delete key removes it. You can also mark "
+ "a rectangle as ignored by hitting the i or END keys when it is selected. Ignored "
+ "rectangles are visually displayed with an X through them. You can remove an image "
+ "entirely by selecting it in the list on the left and pressing alt+d."
+ ,0,0) << endl << endl;
+
+ sout << wrap_string("It is also possible to label object parts by selecting a rectangle and "
+ "then right clicking. A popup menu will appear and you can select a part label. "
+ "Note that you must define the allowable part labels by giving --parts on the "
+ "command line. An example would be '--parts \"leye reye nose mouth\"'."
+ ,0,0) << endl << endl;
+
+ sout << wrap_string("Press the down or s key to select the next image in the list and the up or w "
+ "key to select the previous one.",0,0) << endl << endl;
+
+ sout << wrap_string("Additionally, you can hold ctrl and then scroll the mouse wheel to zoom. A normal left click "
+ "and drag allows you to navigate around the image. Holding ctrl and "
+ "left clicking a rectangle will give it the label from the Next Label field. "
+ "Holding shift + right click and then dragging allows you to move things around. "
+ "Holding ctrl and pressing the up or down keyboard keys will propagate "
+ "rectangle labels from one image to the next and also skip empty images. "
+ "Similarly, holding ctrl+shift will propagate entire boxes via a visual tracking "
+ "algorithm from one image to the next. "
+ "Finally, typing a number on the keyboard will jump you to a specific image.",0,0) << endl << endl;
+
+ sout << wrap_string("You can also toggle image histogram equalization by pressing the e key."
+ ,0,0) << endl;
+
+
+ message_box("About Image Labeler",sout.str());
+}
+
+// ----------------------------------------------------------------------------------------
+
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-03 08:25:44 +0000