// Copyright (C) 2013  Davis E. King (davis@dlib.net)
// License: Boost Software License   See LICENSE.txt for the full license.
#undef DLIB_CROSS_VALIDATE_SEQUENCE_sEGMENTER_ABSTRACT_Hh_
#ifdef DLIB_CROSS_VALIDATE_SEQUENCE_sEGMENTER_ABSTRACT_Hh_

#include "sequence_segmenter_abstract.h"
#include "../matrix.h"

namespace dlib
{

// ----------------------------------------------------------------------------------------

    template <
        typename sequence_segmenter_type,
        typename sequence_type 
        >
    const matrix<double,1,3> test_sequence_segmenter (
        const sequence_segmenter_type& segmenter,
        const std::vector<sequence_type>& samples,
        const std::vector<std::vector<std::pair<unsigned long,unsigned long> > >& segments 
    );
    /*!
        requires
            - is_sequence_segmentation_problem(samples, segments) == true
            - sequence_segmenter_type == dlib::sequence_segmenter or an object with a
              compatible interface.
        ensures
            - Tests segmenter against the given samples and truth segments and returns the
              precision, recall, and F1-score obtained by the segmenter.  That is, the goal
              of the segmenter should be to predict segments[i] given samples[i] as input.
              The test_sequence_segmenter() routine therefore measures how well the
              segmenter is able to perform this task.
            - Returns a row matrix M with the following properties:
                - M(0) == The precision of the segmenter measured against the task of
                  detecting the segments of each sample.  This is a number in the range 0
                  to 1 and represents the fraction of segments output by the segmenter
                  which correspond to true segments for each sample.
                - M(1) == The recall of the segmenter measured against the task of
                  detecting the segments of each sample.  This is a number in the range 0
                  to 1 and represents the fraction of the true segments found by the
                  segmenter. 
                - M(2) == The F1-score for the segmenter.  This is the harmonic mean of
                  M(0) and M(1).
    !*/

// ----------------------------------------------------------------------------------------

    template <
        typename trainer_type,
        typename sequence_type 
        >
    const matrix<double,1,3> cross_validate_sequence_segmenter (
        const trainer_type& trainer,
        const std::vector<sequence_type>& samples,
        const std::vector<std::vector<std::pair<unsigned long,unsigned long> > >& segments,
        const long folds
    );
    /*!
        requires
            - is_sequence_segmentation_problem(samples, segments) == true
            - 1 < folds <= samples.size()
            - trainer_type == dlib::structural_sequence_segmentation_trainer or an object
              with a compatible interface.
        ensures
            - Performs k-fold cross validation by using the given trainer to solve the
              given sequence segmentation problem for the given number of folds.  Each fold
              is tested using the output of the trainer and the results from all folds are
              summarized and returned. 
            - This function returns the precision, recall, and F1-score for the trainer.
              In particular, the output is the same as the output from the
              test_sequence_segmenter() routine defined above.
    !*/

// ----------------------------------------------------------------------------------------

}

#endif // DLIB_CROSS_VALIDATE_SEQUENCE_sEGMENTER_ABSTRACT_Hh_