path: root/ml/dlib/dlib/image_processing/correlation_tracker.h

// Copyright (C) 2015  Davis E. King (davis@dlib.net)
// License: Boost Software License   See LICENSE.txt for the full license.
#ifndef DLIB_CORRELATION_TrACKER_H_
#define DLIB_CORRELATION_TrACKER_H_

#include "correlation_tracker_abstract.h"
#include "../geometry.h"
#include "../matrix.h"
#include "../array2d.h"
#include "../image_transforms/assign_image.h"
#include "../image_transforms/interpolation.h"


namespace dlib
{

// ----------------------------------------------------------------------------------------

    class correlation_tracker
    {
    public:

        explicit correlation_tracker (unsigned long filter_size = 6, 
            unsigned long num_scale_levels = 5, 
            unsigned long scale_window_size = 23,
            double regularizer_space = 0.001,
            double nu_space = 0.025,
            double regularizer_scale = 0.001,
            double nu_scale = 0.025,
            double scale_pyramid_alpha = 1.020
        ) 
            : filter_size(1 << filter_size), num_scale_levels(1 << num_scale_levels),
            scale_window_size(scale_window_size),
            regularizer_space(regularizer_space), nu_space(nu_space), 
            regularizer_scale(regularizer_scale), nu_scale(nu_scale),
            scale_pyramid_alpha(scale_pyramid_alpha)
        {
            // Create the cosine mask used for space filtering.
            mask = make_cosine_mask();

            // Create the cosine mask used for the scale filtering.
            scale_cos_mask.resize(get_num_scale_levels());
            const long max_level = get_num_scale_levels()/2;
            for (unsigned long k = 0; k < get_num_scale_levels(); ++k)
            {
                double dist = std::abs((double)k-max_level)/max_level*pi/2;
                dist = std::min(dist, pi/2);
                scale_cos_mask[k] = std::cos(dist);
            }
        }

        template <typename image_type>
        void start_track (
            const image_type& img,
            const drectangle& p
        )
        {
            DLIB_CASSERT(p.is_empty() == false,
                "\t void correlation_tracker::start_track()"
                << "\n\t You can't give an empty rectangle."
            );

            B.set_size(0,0);

            point_transform_affine tform = inv(make_chip(img, p, F));
            for (unsigned long i = 0; i < F.size(); ++i)
                fft_inplace(F[i]);
            make_target_location_image(tform(center(p)), G);
            A.resize(F.size());
            for (unsigned long i = 0; i < F.size(); ++i)
            {
                A[i] = pointwise_multiply(G, F[i]);
                B += squared(real(F[i]))+squared(imag(F[i]));
            }

            position = p;

            // now do the scale space stuff
            make_scale_space(img, Fs);
            for (unsigned long i = 0; i < Fs.size(); ++i)
                fft_inplace(Fs[i]);
            make_scale_target_location_image(get_num_scale_levels()/2, Gs);
            Bs.set_size(0);
            As.resize(Fs.size());
            for (unsigned long i = 0; i < Fs.size(); ++i)
            {
                As[i] = pointwise_multiply(Gs, Fs[i]);
                Bs += squared(real(Fs[i]))+squared(imag(Fs[i]));
            }
        }


        unsigned long get_filter_size (
        ) const { return filter_size; } 

        unsigned long get_num_scale_levels(
        ) const { return num_scale_levels; }  

        unsigned long get_scale_window_size (
        ) const { return scale_window_size; }

        double get_regularizer_space (
        ) const { return regularizer_space; }
        inline double get_nu_space (
        ) const { return nu_space;}

        double get_regularizer_scale (
        ) const { return regularizer_scale; }
        double get_nu_scale (
        ) const { return nu_scale;}

        drectangle get_position (
        ) const 
        { 
            return position;
        }

        double get_scale_pyramid_alpha (
        ) const { return scale_pyramid_alpha; }


        template <typename image_type>
        double update_noscale(
            const image_type& img,
            const drectangle& guess
        )
        {
            DLIB_CASSERT(get_position().is_empty() == false,
                "\t double correlation_tracker::update()"
                << "\n\t You must call start_track() first before calling update()."
            );


            const point_transform_affine tform = make_chip(img, guess, F);
            for (unsigned long i = 0; i < F.size(); ++i)
                fft_inplace(F[i]);

            // use the current filter to predict the object's location
            G = 0;
            for (unsigned long i = 0; i < F.size(); ++i)
                G += pointwise_multiply(F[i],conj(A[i]));
            G = pointwise_multiply(G, reciprocal(B+get_regularizer_space()));
            ifft_inplace(G);
            const dlib::vector<double,2> pp = max_point_interpolated(real(G));


            // Compute the peak to side lobe ratio.
            const point p = pp;
            running_stats<double> rs;
            const rectangle peak = centered_rect(p, 8,8);
            for (long r = 0; r < G.nr(); ++r)
            {
                for (long c = 0; c < G.nc(); ++c)
                {
                    if (!peak.contains(point(c,r)))
                        rs.add(G(r,c).real());
                }
            }
            const double psr = (G(p.y(),p.x()).real()-rs.mean())/rs.stddev();

            // update the position of the object
            position = translate_rect(guess, tform(pp)-center(guess));

            // now update the position filters
            make_target_location_image(pp, G);
            B *= (1-get_nu_space());
            for (unsigned long i = 0; i < F.size(); ++i)
            {
                A[i] = get_nu_space()*pointwise_multiply(G, F[i]) + (1-get_nu_space())*A[i];
                B += get_nu_space()*(squared(real(F[i]))+squared(imag(F[i])));
            }

            return psr;
        }

        template <typename image_type>
        double update (
            const image_type& img,
            const drectangle& guess
        )
        {
            double psr = update_noscale(img, guess);

            // Now predict the scale change
            make_scale_space(img, Fs);
            for (unsigned long i = 0; i < Fs.size(); ++i)
                fft_inplace(Fs[i]);
            Gs = 0;
            for (unsigned long i = 0; i < Fs.size(); ++i)
                Gs += pointwise_multiply(Fs[i],conj(As[i]));
            Gs = pointwise_multiply(Gs, reciprocal(Bs+get_regularizer_scale()));
            ifft_inplace(Gs);
            const double pos = max_point_interpolated(real(Gs)).y();

            // update the rectangle's scale
            position *= std::pow(get_scale_pyramid_alpha(), pos-(double)get_num_scale_levels()/2);



            // Now update the scale filters
            make_scale_target_location_image(pos, Gs);
            Bs *= (1-get_nu_scale());
            for (unsigned long i = 0; i < Fs.size(); ++i)
            {
                As[i] = get_nu_scale()*pointwise_multiply(Gs, Fs[i]) + (1-get_nu_scale())*As[i];
                Bs += get_nu_scale()*(squared(real(Fs[i]))+squared(imag(Fs[i])));
            }


            return psr;
        }

        template <typename image_type>
        double update_noscale (
            const image_type& img
        )
        {
            return update_noscale(img, get_position());
        }

        template <typename image_type>
        double update(
            const image_type& img
            )
        {
            return update(img, get_position());
        }

    private:

        template <typename image_type>
        void make_scale_space(
            const image_type& img,
            std::vector<matrix<std::complex<double>,0,1> >& Fs
        ) const
        {
            typedef typename image_traits<image_type>::pixel_type pixel_type;

            // Make an image pyramid and put it into the chips array.
            const long chip_size = get_scale_window_size();
            drectangle ppp = position*std::pow(get_scale_pyramid_alpha(), -(double)get_num_scale_levels()/2);
            dlib::array<array2d<pixel_type> > chips;
            std::vector<dlib::vector<double,2> > from_points, to_points;
            from_points.push_back(point(0,0));
            from_points.push_back(point(chip_size-1,0));
            from_points.push_back(point(chip_size-1,chip_size-1));
            for (unsigned long i = 0; i < get_num_scale_levels(); ++i)
            {
                array2d<pixel_type> chip(chip_size,chip_size);

                // pull box into chip
                to_points.clear();
                to_points.push_back(ppp.tl_corner());
                to_points.push_back(ppp.tr_corner());
                to_points.push_back(ppp.br_corner());
                transform_image(img,chip,interpolate_bilinear(),find_affine_transform(from_points, to_points));

                chips.push_back(chip);
                ppp *= get_scale_pyramid_alpha();
            }


            // extract HOG for each chip
            dlib::array<dlib::array<array2d<float> > > hogs(chips.size());
            for (unsigned long i = 0; i < chips.size(); ++i)
            {
                extract_fhog_features(chips[i], hogs[i], 4);
                hogs[i].resize(32);
                assign_image(hogs[i][31], chips[i]);
                assign_image(hogs[i][31], mat(hogs[i][31])/255.0);
            }

            // Now copy the hog features into the Fs outputs and also apply the cosine
            // windowing.
            Fs.resize(hogs[0].size()*hogs[0][0].size());
            unsigned long i = 0; 
            for (long r = 0; r < hogs[0][0].nr(); ++r)
            {
                for (long c = 0; c < hogs[0][0].nc(); ++c)
                {
                    for (unsigned long j = 0; j < hogs[0].size(); ++j)
                    {
                        Fs[i].set_size(hogs.size());
                        for (unsigned long k = 0; k < hogs.size(); ++k)
                        {
                            Fs[i](k) = hogs[k][j][r][c]*scale_cos_mask[k];
                        }
                        ++i;
                    }
                }
            } 
        }

        template <typename image_type>
        point_transform_affine make_chip (
            const image_type& img,
            drectangle p,
            std::vector<matrix<std::complex<double> > >& chip
        ) const
        {
            typedef typename image_traits<image_type>::pixel_type pixel_type;
            array2d<pixel_type> temp;
            const double padding = 1.4;
            const chip_details details(p*padding, chip_dims(get_filter_size(), get_filter_size()));
            extract_image_chip(img, details, temp);


            chip.resize(32);
            dlib::array<array2d<float> > hog;
            extract_fhog_features(temp, hog, 1, 3,3 );
            for (unsigned long i = 0; i < hog.size(); ++i)
                assign_image(chip[i], pointwise_multiply(matrix_cast<double>(mat(hog[i])), mask));

            assign_image(chip[31], temp);
            assign_image(chip[31], pointwise_multiply(mat(chip[31]), mask)/255.0);

            return inv(get_mapping_to_chip(details));
        }

        void make_target_location_image (
            const dlib::vector<double,2>& p,
            matrix<std::complex<double> >& g
        ) const
        {
            g.set_size(get_filter_size(), get_filter_size());
            g = 0;
            rectangle area = centered_rect(p, 21,21).intersect(get_rect(g));
            for (long r = area.top(); r <= area.bottom(); ++r)
            {
                for (long c = area.left(); c <= area.right(); ++c)
                {
                    double dist = length(point(c,r)-p);
                    g(r,c) = std::exp(-dist/3.0);
                }
            }
            fft_inplace(g);
            g = conj(g);
        }


        void make_scale_target_location_image (
            const double scale,
            matrix<std::complex<double>,0,1>& g
        ) const
        {
            g.set_size(get_num_scale_levels());
            for (long i = 0; i < g.size(); ++i)
            {
                double dist = std::pow((i-scale),2.0);
                g(i) = std::exp(-dist/1.000);
            }
            fft_inplace(g);
            g = conj(g);
        }

        matrix<double> make_cosine_mask (
        ) const
        {
            const long size = get_filter_size();
            matrix<double> temp(size,size);
            point cent = center(get_rect(temp));
            for (long r = 0; r < temp.nr(); ++r)
            {
                for (long c = 0; c < temp.nc(); ++c)
                {
                    point delta = point(c,r)-cent;
                    double dist = length(delta)/(size/2.0)*(pi/2);
                    dist = std::min(dist*1.0, pi/2);

                    temp(r,c) = std::cos(dist);
                }
            }
            return temp;
        }


        std::vector<matrix<std::complex<double> > > A, F;
        matrix<double> B;

        std::vector<matrix<std::complex<double>,0,1> > As, Fs;
        matrix<double,0,1> Bs;
        drectangle position;

        matrix<double> mask;
        std::vector<double> scale_cos_mask;

        // G and Gs do not logically contribute to the state of this object.  They are
        // here just so we can void reallocating them over and over.
        matrix<std::complex<double> > G;
        matrix<std::complex<double>,0,1> Gs;

        unsigned long filter_size;
        unsigned long num_scale_levels;
        unsigned long scale_window_size;
        double regularizer_space;
        double nu_space;
        double regularizer_scale;
        double nu_scale;
        double scale_pyramid_alpha;
    };
}

#endif // DLIB_CORRELATION_TrACKER_H_