path: root/ml/dlib/dlib/test/svm.cpp

// Copyright (C) 2006  Davis E. King (davis@dlib.net)
// License: Boost Software License   See LICENSE.txt for the full license.


#include <dlib/matrix.h>
#include <sstream>
#include <string>
#include <cstdlib>
#include <ctime>
#include <vector>
#include "../stl_checked.h"
#include "../array.h"
#include "../rand.h"
#include "checkerboard.h"
#include <dlib/statistics.h>

#include "tester.h"
#include <dlib/svm_threaded.h>


namespace  
{

    using namespace test;
    using namespace dlib;
    using namespace std;

    logger dlog("test.svm");

// ----------------------------------------------------------------------------------------

    void test_clutering (
    )
    {
        dlog << LINFO << "   being test_clutering()";
        // Here we declare that our samples will be 2 dimensional column vectors.  
        typedef matrix<double,2,1> sample_type;

        // Now we are making a typedef for the kind of kernel we want to use.  I picked the
        // radial basis kernel because it only has one parameter and generally gives good
        // results without much fiddling.
        typedef radial_basis_kernel<sample_type> kernel_type;

        // Here we declare an instance of the kcentroid object.  The first argument to the constructor
        // is the kernel we wish to use.  The second is a parameter that determines the numerical 
        // accuracy with which the object will perform part of the learning algorithm.  Generally
        // smaller values give better results but cause the algorithm to run slower.  You just have
        // to play with it to decide what balance of speed and accuracy is right for your problem.
        // Here we have set it to 0.01.
        kcentroid<kernel_type> kc(kernel_type(0.1),0.01);

        // Now we make an instance of the kkmeans object and tell it to use kcentroid objects
        // that are configured with the parameters from the kc object we defined above.
        kkmeans<kernel_type> test(kc);

        std::vector<sample_type> samples;
        std::vector<sample_type> initial_centers;

        sample_type m;

        dlib::rand rnd;

        print_spinner();
        // we will make 50 points from each class
        const long num = 50;

        // make some samples near the origin
        double radius = 0.5;
        for (long i = 0; i < num; ++i)
        {
            double sign = 1;
            if (rnd.get_random_double() < 0.5)
                sign = -1;
            m(0) = 2*radius*rnd.get_random_double()-radius;
            m(1) = sign*sqrt(radius*radius - m(0)*m(0));

            // add this sample to our set of samples we will run k-means 
            samples.push_back(m);
        }

        // make some samples in a circle around the origin but far away
        radius = 10.0;
        for (long i = 0; i < num; ++i)
        {
            double sign = 1;
            if (rnd.get_random_double() < 0.5)
                sign = -1;
            m(0) = 2*radius*rnd.get_random_double()-radius;
            m(1) = sign*sqrt(radius*radius - m(0)*m(0));

            // add this sample to our set of samples we will run k-means 
            samples.push_back(m);
        }

        // make some samples in a circle around the point (25,25) 
        radius = 4.0;
        for (long i = 0; i < num; ++i)
        {
            double sign = 1;
            if (rnd.get_random_double() < 0.5)
                sign = -1;
            m(0) = 2*radius*rnd.get_random_double()-radius;
            m(1) = sign*sqrt(radius*radius - m(0)*m(0));

            // translate this point away from the origin
            m(0) += 25;
            m(1) += 25;

            // add this sample to our set of samples we will run k-means 
            samples.push_back(m);
        }
        print_spinner();

        // tell the kkmeans object we made that we want to run k-means with k set to 3. 
        // (i.e. we want 3 clusters)
        test.set_number_of_centers(3);

        // You need to pick some initial centers for the k-means algorithm.  So here
        // we will use the dlib::pick_initial_centers() function which tries to find
        // n points that are far apart (basically).  
        pick_initial_centers(3, initial_centers, samples, test.get_kernel());

        print_spinner();
        // now run the k-means algorithm on our set of samples.  
        test.train(samples,initial_centers);
        print_spinner();

        const unsigned long class1 = test(samples[0]);
        const unsigned long class2 = test(samples[num]);
        const unsigned long class3 = test(samples[2*num]);
        // now loop over all our samples and print out their predicted class.  In this example
        // all points are correctly identified.
        for (unsigned long i = 0; i < samples.size()/3; ++i)
        {
            DLIB_TEST(test(samples[i]) == class1);
            DLIB_TEST(test(samples[i+num]) == class2);
            DLIB_TEST(test(samples[i+2*num]) == class3);
        }

        dlog << LINFO << "   end test_clutering()";
    }

// ----------------------------------------------------------------------------------------

    // Here is the sinc function we will be trying to learn with the krls
    // object.
    double sinc(double x)
    {
        if (x == 0)
            return 1;
        return sin(x)/x;
    }


    void test_regression (
    )
    {
        dlog << LINFO << "   being test_regression()";
        // Here we declare that our samples will be 1 dimensional column vectors.  The reason for
        // using a matrix here is that in general you can use N dimensional vectors as inputs to the
        // krls object.  But here we only have 1 dimension to make the example simple.
        typedef matrix<double,1,1> sample_type;

        // Now we are making a typedef for the kind of kernel we want to use.  I picked the
        // radial basis kernel because it only has one parameter and generally gives good
        // results without much fiddling.
        typedef radial_basis_kernel<sample_type> kernel_type;

        // Here we declare an instance of the krls object.  The first argument to the constructor
        // is the kernel we wish to use.  The second is a parameter that determines the numerical 
        // accuracy with which the object will perform part of the regression algorithm.  Generally
        // smaller values give better results but cause the algorithm to run slower.  You just have
        // to play with it to decide what balance of speed and accuracy is right for your problem.
        // Here we have set it to 0.001.
        krls<kernel_type> test(kernel_type(0.1),0.001);
        rvm_regression_trainer<kernel_type> rvm_test;
        rvm_test.set_kernel(test.get_kernel());

        krr_trainer<kernel_type> krr_test;
        krr_test.set_kernel(test.get_kernel());

        svr_trainer<kernel_type> svr_test;
        svr_test.set_kernel(test.get_kernel());
        svr_test.set_epsilon_insensitivity(0.0001);
        svr_test.set_c(10);

        rbf_network_trainer<kernel_type> rbf_test;
        rbf_test.set_kernel(test.get_kernel());
        rbf_test.set_num_centers(13);

        print_spinner();
        std::vector<sample_type> samples;
        std::vector<sample_type> samples2;
        std::vector<double> labels;
        std::vector<double> labels2;
        // now we train our object on a few samples of the sinc function.
        sample_type m;
        for (double x = -10; x <= 5; x += 0.6)
        {
            m(0) = x;
            test.train(m, sinc(x));

            samples.push_back(m);
            samples2.push_back(m);
            labels.push_back(sinc(x));
            labels2.push_back(2);
        }

        print_spinner();
        decision_function<kernel_type> test2 = rvm_test.train(samples, labels);
        print_spinner();
        decision_function<kernel_type> test3 = rbf_test.train(samples, labels);
        print_spinner();
        decision_function<kernel_type> test4 = krr_test.train(samples, labels);
        print_spinner();
        decision_function<kernel_type> test5 = svr_test.train(samples, labels);
        print_spinner();

        // now we output the value of the sinc function for a few test points as well as the 
        // value predicted by krls object.
        m(0) = 2.5; dlog << LDEBUG << "krls: " << sinc(m(0)) << "   " << test(m); DLIB_TEST(abs(sinc(m(0)) - test(m)) < 0.01);
        m(0) = 0.1; dlog << LDEBUG << "krls: " << sinc(m(0)) << "   " << test(m); DLIB_TEST(abs(sinc(m(0)) - test(m)) < 0.01);
        m(0) = -4;  dlog << LDEBUG << "krls: " << sinc(m(0)) << "   " << test(m); DLIB_TEST(abs(sinc(m(0)) - test(m)) < 0.01);
        m(0) = 5.0; dlog << LDEBUG << "krls: " << sinc(m(0)) << "   " << test(m); DLIB_TEST(abs(sinc(m(0)) - test(m)) < 0.01);

        m(0) = 2.5; dlog << LDEBUG << "rvm: " << sinc(m(0)) << "   " << test2(m); DLIB_TEST(abs(sinc(m(0)) - test2(m)) < 0.01);
        m(0) = 0.1; dlog << LDEBUG << "rvm: " << sinc(m(0)) << "   " << test2(m); DLIB_TEST(abs(sinc(m(0)) - test2(m)) < 0.01);
        m(0) = -4;  dlog << LDEBUG << "rvm: " << sinc(m(0)) << "   " << test2(m); DLIB_TEST(abs(sinc(m(0)) - test2(m)) < 0.01);
        m(0) = 5.0; dlog << LDEBUG << "rvm: " << sinc(m(0)) << "   " << test2(m); DLIB_TEST(abs(sinc(m(0)) - test2(m)) < 0.01);

        m(0) = 2.5; dlog << LDEBUG << "rbf: " << sinc(m(0)) << "   " << test3(m); DLIB_TEST(abs(sinc(m(0)) - test3(m)) < 0.01);
        m(0) = 0.1; dlog << LDEBUG << "rbf: " << sinc(m(0)) << "   " << test3(m); DLIB_TEST(abs(sinc(m(0)) - test3(m)) < 0.01);
        m(0) = -4;  dlog << LDEBUG << "rbf: " << sinc(m(0)) << "   " << test3(m); DLIB_TEST(abs(sinc(m(0)) - test3(m)) < 0.01);
        m(0) = 5.0; dlog << LDEBUG << "rbf: " << sinc(m(0)) << "   " << test3(m); DLIB_TEST(abs(sinc(m(0)) - test3(m)) < 0.01);

        m(0) = 2.5; dlog << LDEBUG << "krr: " << sinc(m(0)) << "   " << test4(m); DLIB_TEST(abs(sinc(m(0)) - test4(m)) < 0.01);
        m(0) = 0.1; dlog << LDEBUG << "krr: " << sinc(m(0)) << "   " << test4(m); DLIB_TEST(abs(sinc(m(0)) - test4(m)) < 0.01);
        m(0) = -4;  dlog << LDEBUG << "krr: " << sinc(m(0)) << "   " << test4(m); DLIB_TEST(abs(sinc(m(0)) - test4(m)) < 0.01);
        m(0) = 5.0; dlog << LDEBUG << "krr: " << sinc(m(0)) << "   " << test4(m); DLIB_TEST(abs(sinc(m(0)) - test4(m)) < 0.01);

        m(0) = 2.5; dlog << LDEBUG << "svr: " << sinc(m(0)) << "   " << test5(m); DLIB_TEST(abs(sinc(m(0)) - test5(m)) < 0.01);
        m(0) = 0.1; dlog << LDEBUG << "svr: " << sinc(m(0)) << "   " << test5(m); DLIB_TEST(abs(sinc(m(0)) - test5(m)) < 0.01);
        m(0) = -4;  dlog << LDEBUG << "svr: " << sinc(m(0)) << "   " << test5(m); DLIB_TEST(abs(sinc(m(0)) - test5(m)) < 0.01);
        m(0) = 5.0; dlog << LDEBUG << "svr: " << sinc(m(0)) << "   " << test5(m); DLIB_TEST(abs(sinc(m(0)) - test5(m)) < 0.01);


        randomize_samples(samples, labels);
        dlog << LINFO << "KRR MSE and R-squared: "<< cross_validate_regression_trainer(krr_test, samples, labels, 6);
        dlog << LINFO << "SVR MSE and R-squared: "<< cross_validate_regression_trainer(svr_test, samples, labels, 6);
        matrix<double,1,4> cv = cross_validate_regression_trainer(krr_test, samples, labels, 6);
        DLIB_TEST(cv(0) < 1e-4);
        DLIB_TEST(cv(1) > 0.99);
        cv = cross_validate_regression_trainer(svr_test, samples, labels, 6);
        DLIB_TEST(cv(0) < 1e-4);
        DLIB_TEST(cv(1) > 0.99);




        randomize_samples(samples2, labels2);
        dlog << LINFO << "KRR MSE and R-squared: "<< cross_validate_regression_trainer(krr_test, samples2, labels2, 6);
        dlog << LINFO << "SVR MSE and R-squared: "<< cross_validate_regression_trainer(svr_test, samples2, labels2, 6);
        cv = cross_validate_regression_trainer(krr_test, samples2, labels2, 6);
        DLIB_TEST(cv(0) < 1e-4);
        cv = cross_validate_regression_trainer(svr_test, samples2, labels2, 6);
        DLIB_TEST(cv(0) < 1e-4);

        dlog << LINFO << "   end test_regression()";
    }

// ----------------------------------------------------------------------------------------

    void test_anomaly_detection (
    ) 
    {
        dlog << LINFO << "   begin test_anomaly_detection()";
        // Here we declare that our samples will be 2 dimensional column vectors.  
        typedef matrix<double,2,1> sample_type;

        // Now we are making a typedef for the kind of kernel we want to use.  I picked the
        // radial basis kernel because it only has one parameter and generally gives good
        // results without much fiddling.
        typedef radial_basis_kernel<sample_type> kernel_type;

        // Here we declare an instance of the kcentroid object.  The first argument to the constructor
        // is the kernel we wish to use.  The second is a parameter that determines the numerical 
        // accuracy with which the object will perform part of the learning algorithm.  Generally
        // smaller values give better results but cause the algorithm to run slower.  You just have
        // to play with it to decide what balance of speed and accuracy is right for your problem.
        // Here we have set it to 0.01.
        kcentroid<kernel_type> test(kernel_type(0.1),0.01);


        svm_one_class_trainer<kernel_type> one_class_trainer;
        one_class_trainer.set_nu(0.4);
        one_class_trainer.set_kernel(kernel_type(0.2));

        std::vector<sample_type> samples;

        // now we train our object on a few samples of the sinc function.
        sample_type m;
        for (double x = -15; x <= 8; x += 1)
        {
            m(0) = x;
            m(1) = sinc(x);
            test.train(m);
            samples.push_back(m);
        }

        decision_function<kernel_type> df = one_class_trainer.train(samples);

        running_stats<double> rs;

        // Now lets output the distance from the centroid to some points that are from the sinc function.
        // These numbers should all be similar.  We will also calculate the statistics of these numbers
        // by accumulating them into the running_stats object called rs.  This will let us easily
        // find the mean and standard deviation of the distances for use below.
        dlog << LDEBUG << "Points that are on the sinc function:\n";
        m(0) = -1.5; m(1) = sinc(m(0)); dlog << LDEBUG << "   " << test(m);  rs.add(test(m));
        m(0) = -1.5; m(1) = sinc(m(0)); dlog << LDEBUG << "   " << test(m);  rs.add(test(m));
        m(0) = -0;   m(1) = sinc(m(0)); dlog << LDEBUG << "   " << test(m);  rs.add(test(m));
        m(0) = -0.5; m(1) = sinc(m(0)); dlog << LDEBUG << "   " << test(m);  rs.add(test(m));
        m(0) = -4.1; m(1) = sinc(m(0)); dlog << LDEBUG << "   " << test(m);  rs.add(test(m));
        m(0) = -1.5; m(1) = sinc(m(0)); dlog << LDEBUG << "   " << test(m);  rs.add(test(m));
        m(0) = -0.5; m(1) = sinc(m(0)); dlog << LDEBUG << "   " << test(m);  rs.add(test(m));

        m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
        m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
        m(0) = -0;   m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
        m(0) = -0.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
        m(0) = -4.1; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
        m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));
        m(0) = -0.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(rs.scale(test(m)) < 2, rs.scale(test(m)));

        const double thresh = 0.01;
        m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
        m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
        m(0) = -0;   m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
        m(0) = -0.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
        m(0) = -4.1; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
        m(0) = -1.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));
        m(0) = -0.5; m(1) = sinc(m(0)); DLIB_TEST_MSG(df(m)+thresh > 0, df(m));

        dlog << LDEBUG;
        // Lets output the distance from the centroid to some points that are NOT from the sinc function.
        // These numbers should all be significantly bigger than previous set of numbers.  We will also
        // use the rs.scale() function to find out how many standard deviations they are away from the 
        // mean of the test points from the sinc function.  So in this case our criterion for "significantly bigger"
        // is > 3 or 4 standard deviations away from the above points that actually are on the sinc function.
        dlog << LDEBUG << "Points that are NOT on the sinc function:\n";
        m(0) = -1.5; m(1) = sinc(m(0))+4;   
        dlog << LDEBUG << "   " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
        DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
        DLIB_TEST_MSG(df(m) + thresh < 0, df(m));

        m(0) = -1.5; m(1) = sinc(m(0))+3;   
        dlog << LDEBUG << "   " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
        DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
        DLIB_TEST_MSG(df(m) + thresh < 0, df(m));

        m(0) = -0;   m(1) = -sinc(m(0));    
        dlog << LDEBUG << "   " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
        DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
        DLIB_TEST_MSG(df(m) + thresh < 0, df(m));

        m(0) = -0.5; m(1) = -sinc(m(0));    
        dlog << LDEBUG << "   " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
        DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
        DLIB_TEST_MSG(df(m) + thresh < 0, df(m));

        m(0) = -4.1; m(1) = sinc(m(0))+2;   
        dlog << LDEBUG << "   " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
        DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
        DLIB_TEST_MSG(df(m) + thresh < 0, df(m));

        m(0) = -1.5; m(1) = sinc(m(0))+0.9; 
        dlog << LDEBUG << "   " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
        DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
        DLIB_TEST_MSG(df(m) + thresh < 0, df(m));

        m(0) = -0.5; m(1) = sinc(m(0))+1;   
        dlog << LDEBUG << "   " << test(m) << " is " << rs.scale(test(m)) << " standard deviations from sinc.";
        DLIB_TEST_MSG(rs.scale(test(m)) > 6, rs.scale(test(m)));
        DLIB_TEST_MSG(df(m) + thresh < 0, df(m));

        dlog << LINFO << "   end test_anomaly_detection()";
    }

// ----------------------------------------------------------------------------------------

    void unittest_binary_classification (
    )
    /*!
        ensures
            - runs tests on the svm stuff compliance with the specs
    !*/
    {        
        dlog << LINFO << "   begin unittest_binary_classification()";
        print_spinner();


        typedef double scalar_type;
        typedef matrix<scalar_type,2,1> sample_type;

        std::vector<sample_type> x;
        std::vector<matrix<double,0,1> > x_linearized;
        std::vector<scalar_type> y;

        get_checkerboard_problem(x,y, 300, 2);
        const scalar_type gamma = 1;

        typedef radial_basis_kernel<sample_type> kernel_type;

        rbf_network_trainer<kernel_type> rbf_trainer;
        rbf_trainer.set_kernel(kernel_type(gamma));
        rbf_trainer.set_num_centers(100);

        rvm_trainer<kernel_type> rvm_trainer;
        rvm_trainer.set_kernel(kernel_type(gamma));

        krr_trainer<kernel_type> krr_trainer;
        krr_trainer.use_classification_loss_for_loo_cv();
        krr_trainer.set_kernel(kernel_type(gamma));

        svm_pegasos<kernel_type> pegasos_trainer;
        pegasos_trainer.set_kernel(kernel_type(gamma));
        pegasos_trainer.set_lambda(0.00001);


        svm_c_ekm_trainer<kernel_type> ocas_ekm_trainer;
        ocas_ekm_trainer.set_kernel(kernel_type(gamma));
        ocas_ekm_trainer.set_c(100000);

        svm_nu_trainer<kernel_type> trainer;
        trainer.set_kernel(kernel_type(gamma));
        trainer.set_nu(0.05);

        svm_c_trainer<kernel_type> c_trainer;
        c_trainer.set_kernel(kernel_type(gamma));
        c_trainer.set_c(100);

        svm_c_linear_trainer<linear_kernel<matrix<double,0,1> > > lin_trainer;
        lin_trainer.set_c(100000);
        // use an ekm to linearize this dataset so we can use it with the lin_trainer
        empirical_kernel_map<kernel_type> ekm;
        ekm.load(kernel_type(gamma), x);
        for (unsigned long i = 0; i < x.size(); ++i)
            x_linearized.push_back(ekm.project(x[i]));


        print_spinner();
        matrix<scalar_type> rvm_cv = cross_validate_trainer_threaded(rvm_trainer, x,y, 4, 2);
        print_spinner();
        matrix<scalar_type> krr_cv = cross_validate_trainer_threaded(krr_trainer, x,y, 4, 2);
        print_spinner();
        matrix<scalar_type> svm_cv = cross_validate_trainer(trainer, x,y, 4);
        print_spinner();
        matrix<scalar_type> svm_c_cv = cross_validate_trainer(c_trainer, x,y, 4);
        print_spinner();
        matrix<scalar_type> rbf_cv = cross_validate_trainer_threaded(rbf_trainer, x,y, 10, 2);
        print_spinner();
        matrix<scalar_type> lin_cv = cross_validate_trainer_threaded(lin_trainer, x_linearized, y, 4, 2);
        print_spinner();
        matrix<scalar_type> ocas_ekm_cv = cross_validate_trainer_threaded(ocas_ekm_trainer, x, y, 4, 2);
        print_spinner();
        ocas_ekm_trainer.set_basis(randomly_subsample(x, 300));
        matrix<scalar_type> ocas_ekm_cv2 = cross_validate_trainer_threaded(ocas_ekm_trainer, x, y, 4, 2);
        print_spinner();
        matrix<scalar_type> peg_cv = cross_validate_trainer_threaded(batch(pegasos_trainer,1.0), x,y, 4, 2);
        print_spinner();
        matrix<scalar_type> peg_c_cv = cross_validate_trainer_threaded(batch_cached(pegasos_trainer,1.0), x,y, 4, 2);
        print_spinner();

        dlog << LDEBUG << "rvm cv:        " << rvm_cv;
        dlog << LDEBUG << "krr cv:        " << krr_cv;
        dlog << LDEBUG << "nu-svm cv:     " << svm_cv;
        dlog << LDEBUG << "C-svm cv:      " << svm_c_cv;
        dlog << LDEBUG << "rbf cv:        " << rbf_cv;
        dlog << LDEBUG << "lin cv:        " << lin_cv;
        dlog << LDEBUG << "ocas_ekm cv:   " << ocas_ekm_cv;
        dlog << LDEBUG << "ocas_ekm cv2:  " << ocas_ekm_cv2;
        dlog << LDEBUG << "peg cv:        " << peg_cv;
        dlog << LDEBUG << "peg cached cv: " << peg_c_cv;

        // make sure the cached version of pegasos computes the same result
        DLIB_TEST_MSG(sum(abs(peg_cv - peg_c_cv)) < std::sqrt(std::numeric_limits<double>::epsilon()),
                      sum(abs(peg_cv - peg_c_cv)) << "   \n" << peg_cv << peg_c_cv  );

        DLIB_TEST_MSG(mean(rvm_cv) > 0.9, rvm_cv);
        DLIB_TEST_MSG(mean(krr_cv) > 0.9, krr_cv);
        DLIB_TEST_MSG(mean(svm_cv) > 0.9, svm_cv);
        DLIB_TEST_MSG(mean(svm_c_cv) > 0.9, svm_c_cv);
        DLIB_TEST_MSG(mean(rbf_cv) > 0.9, rbf_cv);
        DLIB_TEST_MSG(mean(lin_cv) > 0.9, lin_cv);
        DLIB_TEST_MSG(mean(peg_cv) > 0.9, peg_cv);
        DLIB_TEST_MSG(mean(peg_c_cv) > 0.9, peg_c_cv);
        DLIB_TEST_MSG(mean(ocas_ekm_cv) > 0.9, ocas_ekm_cv);
        DLIB_TEST_MSG(mean(ocas_ekm_cv2) > 0.9, ocas_ekm_cv2);

        const long num_sv = trainer.train(x,y).basis_vectors.size();
        print_spinner();
        const long num_rv = rvm_trainer.train(x,y).basis_vectors.size();
        print_spinner();
        dlog << LDEBUG << "num sv: " << num_sv;
        dlog << LDEBUG << "num rv: " << num_rv;
        print_spinner();
        ocas_ekm_trainer.clear_basis();
        const long num_bv = ocas_ekm_trainer.train(x,y).basis_vectors.size();
        dlog << LDEBUG << "num ekm bv: " << num_bv;

        DLIB_TEST(num_rv <= 17);
        DLIB_TEST_MSG(num_sv <= 45, num_sv);
        DLIB_TEST_MSG(num_bv <= 45, num_bv);

        decision_function<kernel_type> df = reduced2(trainer, 19).train(x,y);
        print_spinner();

        matrix<scalar_type> svm_reduced_error = test_binary_decision_function(df, x, y);
        print_spinner();
        dlog << LDEBUG << "svm reduced test error: " << svm_reduced_error;
        dlog << LDEBUG << "svm reduced num sv: " << df.basis_vectors.size();
        DLIB_TEST(mean(svm_reduced_error) > 0.9);

        svm_cv = cross_validate_trainer(reduced(trainer,30), x,y, 4);
        dlog << LDEBUG << "svm reduced cv: " << svm_cv;
        DLIB_TEST_MSG(mean(svm_cv) > 0.9, svm_cv);

        DLIB_TEST(df.basis_vectors.size() <= 19);
        dlog << LINFO << "   end unittest_binary_classification()";
    }

// ----------------------------------------------------------------------------------------

    template <typename kernel_type>
    struct kernel_der_obj
    {
        typename kernel_type::sample_type x;
        kernel_type k;

        double operator()(const typename kernel_type::sample_type& y) const { return k(x,y); }
    };


    template <typename kernel_type>
    void test_kernel_derivative (
        const kernel_type& k,
        const typename kernel_type::sample_type& x, 
        const typename kernel_type::sample_type& y 
    )
    {
        kernel_der_obj<kernel_type> obj;
        obj.x = x;
        obj.k = k;
        kernel_derivative<kernel_type> der(obj.k);
        DLIB_TEST(dlib::equal(derivative(obj)(y) , der(obj.x,y), 1e-5));
    }

    void test_kernel_derivative (
    )
    {
        typedef matrix<double, 2, 1> sample_type;

        sigmoid_kernel<sample_type> k1;
        radial_basis_kernel<sample_type> k2;
        linear_kernel<sample_type> k3;
        polynomial_kernel<sample_type> k4(2,3,4);

        offset_kernel<sigmoid_kernel<sample_type> > k5;
        offset_kernel<radial_basis_kernel<sample_type> > k6;

        dlib::rand rnd;

        sample_type x, y;
        for (int i = 0; i < 10; ++i)
        {
            x = randm(2,1,rnd);
            y = randm(2,1,rnd);
            test_kernel_derivative(k1, x, y);
            test_kernel_derivative(k2, x, y);
            test_kernel_derivative(k3, x, y);
            test_kernel_derivative(k4, x, y);
            test_kernel_derivative(k5, x, y);
            test_kernel_derivative(k6, x, y);
        }
    }

// ----------------------------------------------------------------------------------------

    void test_svm_trainer2()
    {
        typedef matrix<double, 2, 1> sample_type;
        typedef linear_kernel<sample_type> kernel_type;


        std::vector<sample_type> samples;
        std::vector<double> labels;

        sample_type samp;
        samp(0) = 1;
        samp(1) = 1;
        samples.push_back(samp);
        labels.push_back(+1);

        samp(0) = 1;
        samp(1) = 2;
        samples.push_back(samp);
        labels.push_back(-1);

        svm_c_trainer<kernel_type> trainer;

        decision_function<kernel_type> df = trainer.train(samples, labels);

        samp(0) = 1;
        samp(1) = 1;
        dlog << LINFO << "test +1 : "<< df(samp);
        DLIB_TEST(df(samp) > 0);
        samp(0) = 1;
        samp(1) = 2;
        dlog << LINFO << "test -1 : "<< df(samp);
        DLIB_TEST(df(samp) < 0);

        svm_nu_trainer<kernel_type> trainer2;
        df = trainer2.train(samples, labels);

        samp(0) = 1;
        samp(1) = 1;
        dlog << LINFO << "test +1 : "<< df(samp);
        DLIB_TEST(df(samp) > 0);
        samp(0) = 1;
        samp(1) = 2;
        dlog << LINFO << "test -1 : "<< df(samp);
        DLIB_TEST(df(samp) < 0);

    }

// ----------------------------------------------------------------------------------------

    class svm_tester : public tester
    {
    public:
        svm_tester (
        ) :
            tester ("test_svm",
                    "Runs tests on the svm/kernel algorithm components.")
        {}

        void perform_test (
        )
        {
            test_kernel_derivative();
            unittest_binary_classification();
            test_clutering();
            test_regression();
            test_anomaly_detection();
            test_svm_trainer2();
        }
    } a;

}