diff options
Diffstat (limited to '')
-rw-r--r-- | ml/dlib/examples/dnn_introduction2_ex.cpp | 388 |
1 files changed, 388 insertions, 0 deletions
diff --git a/ml/dlib/examples/dnn_introduction2_ex.cpp b/ml/dlib/examples/dnn_introduction2_ex.cpp new file mode 100644 index 000000000..70b6edee7 --- /dev/null +++ b/ml/dlib/examples/dnn_introduction2_ex.cpp @@ -0,0 +1,388 @@ +// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt +/* + This is an example illustrating the use of the deep learning tools from the + dlib C++ Library. I'm assuming you have already read the dnn_introduction_ex.cpp + example. So in this example program I'm going to go over a number of more + advanced parts of the API, including: + - Using multiple GPUs + - Training on large datasets that don't fit in memory + - Defining large networks + - Accessing and configuring layers in a network +*/ + +#include <dlib/dnn.h> +#include <iostream> +#include <dlib/data_io.h> + +using namespace std; +using namespace dlib; + +// ---------------------------------------------------------------------------------------- + +// Let's start by showing how you can conveniently define large and complex +// networks. The most important tool for doing this are C++'s alias templates. +// These let us define new layer types that are combinations of a bunch of other +// layers. These will form the building blocks for more complex networks. + +// So let's begin by defining the building block of a residual network (see +// Figure 2 in Deep Residual Learning for Image Recognition by He, Zhang, Ren, +// and Sun). We are going to decompose the residual block into a few alias +// statements. First, we define the core block. + +// Here we have parameterized the "block" layer on a BN layer (nominally some +// kind of batch normalization), the number of filter outputs N, and the stride +// the block operates at. +template < + int N, + template <typename> class BN, + int stride, + typename SUBNET + > +using block = BN<con<N,3,3,1,1,relu<BN<con<N,3,3,stride,stride,SUBNET>>>>>; + +// Next, we need to define the skip layer mechanism used in the residual network +// paper. They create their blocks by adding the input tensor to the output of +// each block. So we define an alias statement that takes a block and wraps it +// with this skip/add structure. + +// Note the tag layer. This layer doesn't do any computation. It exists solely +// so other layers can refer to it. In this case, the add_prev1 layer looks for +// the tag1 layer and will take the tag1 output and add it to the input of the +// add_prev1 layer. This combination allows us to implement skip and residual +// style networks. We have also set the block stride to 1 in this statement. +// The significance of that is explained next. +template < + template <int,template<typename>class,int,typename> class block, + int N, + template<typename>class BN, + typename SUBNET + > +using residual = add_prev1<block<N,BN,1,tag1<SUBNET>>>; + +// Some residual blocks do downsampling. They do this by using a stride of 2 +// instead of 1. However, when downsampling we need to also take care to +// downsample the part of the network that adds the original input to the output +// or the sizes won't make sense (the network will still run, but the results +// aren't as good). So here we define a downsampling version of residual. In +// it, we make use of the skip1 layer. This layer simply outputs whatever is +// output by the tag1 layer. Therefore, the skip1 layer (there are also skip2, +// skip3, etc. in dlib) allows you to create branching network structures. + +// residual_down creates a network structure like this: +/* + input from SUBNET + / \ + / \ + block downsample(using avg_pool) + \ / + \ / + add tensors (using add_prev2 which adds the output of tag2 with avg_pool's output) + | + output +*/ +template < + template <int,template<typename>class,int,typename> class block, + int N, + template<typename>class BN, + typename SUBNET + > +using residual_down = add_prev2<avg_pool<2,2,2,2,skip1<tag2<block<N,BN,2,tag1<SUBNET>>>>>>; + + + +// Now we can define 4 different residual blocks we will use in this example. +// The first two are non-downsampling residual blocks while the last two +// downsample. Also, res and res_down use batch normalization while ares and +// ares_down have had the batch normalization replaced with simple affine +// layers. We will use the affine version of the layers when testing our +// networks. +template <typename SUBNET> using res = relu<residual<block,8,bn_con,SUBNET>>; +template <typename SUBNET> using ares = relu<residual<block,8,affine,SUBNET>>; +template <typename SUBNET> using res_down = relu<residual_down<block,8,bn_con,SUBNET>>; +template <typename SUBNET> using ares_down = relu<residual_down<block,8,affine,SUBNET>>; + + + +// Now that we have these convenient aliases, we can define a residual network +// without a lot of typing. Note the use of a repeat layer. This special layer +// type allows us to type repeat<9,res,SUBNET> instead of +// res<res<res<res<res<res<res<res<res<SUBNET>>>>>>>>>. It will also prevent +// the compiler from complaining about super deep template nesting when creating +// large networks. +const unsigned long number_of_classes = 10; +using net_type = loss_multiclass_log<fc<number_of_classes, + avg_pool_everything< + res<res<res<res_down< + repeat<9,res, // repeat this layer 9 times + res_down< + res< + input<matrix<unsigned char>> + >>>>>>>>>>; + + +// And finally, let's define a residual network building block that uses +// parametric ReLU units instead of regular ReLU. +template <typename SUBNET> +using pres = prelu<add_prev1<bn_con<con<8,3,3,1,1,prelu<bn_con<con<8,3,3,1,1,tag1<SUBNET>>>>>>>>; + +// ---------------------------------------------------------------------------------------- + +int main(int argc, char** argv) try +{ + if (argc != 2) + { + cout << "This example needs the MNIST dataset to run!" << endl; + cout << "You can get MNIST from http://yann.lecun.com/exdb/mnist/" << endl; + cout << "Download the 4 files that comprise the dataset, decompress them, and" << endl; + cout << "put them in a folder. Then give that folder as input to this program." << endl; + return 1; + } + + std::vector<matrix<unsigned char>> training_images; + std::vector<unsigned long> training_labels; + std::vector<matrix<unsigned char>> testing_images; + std::vector<unsigned long> testing_labels; + load_mnist_dataset(argv[1], training_images, training_labels, testing_images, testing_labels); + + + // dlib uses cuDNN under the covers. One of the features of cuDNN is the + // option to use slower methods that use less RAM or faster methods that use + // a lot of RAM. If you find that you run out of RAM on your graphics card + // then you can call this function and we will request the slower but more + // RAM frugal cuDNN algorithms. + set_dnn_prefer_smallest_algorithms(); + + + // Create a network as defined above. This network will produce 10 outputs + // because that's how we defined net_type. However, fc layers can have the + // number of outputs they produce changed at runtime. + net_type net; + // So if you wanted to use the same network but override the number of + // outputs at runtime you can do so like this: + net_type net2(num_fc_outputs(15)); + + // Now, let's imagine we wanted to replace some of the relu layers with + // prelu layers. We might do it like this: + using net_type2 = loss_multiclass_log<fc<number_of_classes, + avg_pool_everything< + pres<res<res<res_down< // 2 prelu layers here + tag4<repeat<9,pres, // 9 groups, each containing 2 prelu layers + res_down< + res< + input<matrix<unsigned char>> + >>>>>>>>>>>; + + // prelu layers have a floating point parameter. If you want to set it to + // something other than its default value you can do so like this: + net_type2 pnet(prelu_(0.2), + prelu_(0.25), + repeat_group(prelu_(0.3),prelu_(0.4)) // Initialize all the prelu instances in the repeat + // layer. repeat_group() is needed to group the + // things that are part of repeat's block. + ); + // As you can see, a network will greedily assign things given to its + // constructor to the layers inside itself. The assignment is done in the + // order the layers are defined, but it will skip layers where the + // assignment doesn't make sense. + + // Now let's print the details of the pnet to the screen and inspect it. + cout << "The pnet has " << pnet.num_layers << " layers in it." << endl; + cout << pnet << endl; + // These print statements will output this (I've truncated it since it's + // long, but you get the idea): + /* + The pnet has 131 layers in it. + layer<0> loss_multiclass_log + layer<1> fc (num_outputs=10) learning_rate_mult=1 weight_decay_mult=1 bias_learning_rate_mult=1 bias_weight_decay_mult=0 + layer<2> avg_pool (nr=0, nc=0, stride_y=1, stride_x=1, padding_y=0, padding_x=0) + layer<3> prelu (initial_param_value=0.2) + layer<4> add_prev1 + layer<5> bn_con eps=1e-05 learning_rate_mult=1 weight_decay_mult=0 bias_learning_rate_mult=1 bias_weight_decay_mult=1 + layer<6> con (num_filters=8, nr=3, nc=3, stride_y=1, stride_x=1, padding_y=1, padding_x=1) learning_rate_mult=1 weight_decay_mult=1 bias_learning_rate_mult=1 bias_weight_decay_mult=0 + layer<7> prelu (initial_param_value=0.25) + layer<8> bn_con eps=1e-05 learning_rate_mult=1 weight_decay_mult=0 bias_learning_rate_mult=1 bias_weight_decay_mult=1 + layer<9> con (num_filters=8, nr=3, nc=3, stride_y=1, stride_x=1, padding_y=1, padding_x=1) learning_rate_mult=1 weight_decay_mult=1 bias_learning_rate_mult=1 bias_weight_decay_mult=0 + layer<10> tag1 + ... + layer<34> relu + layer<35> bn_con eps=1e-05 learning_rate_mult=1 weight_decay_mult=0 bias_learning_rate_mult=1 bias_weight_decay_mult=1 + layer<36> con (num_filters=8, nr=3, nc=3, stride_y=2, stride_x=2, padding_y=0, padding_x=0) learning_rate_mult=1 weight_decay_mult=1 bias_learning_rate_mult=1 bias_weight_decay_mult=0 + layer<37> tag1 + layer<38> tag4 + layer<39> prelu (initial_param_value=0.3) + layer<40> add_prev1 + layer<41> bn_con eps=1e-05 learning_rate_mult=1 weight_decay_mult=0 bias_learning_rate_mult=1 bias_weight_decay_mult=1 + ... + layer<118> relu + layer<119> bn_con eps=1e-05 learning_rate_mult=1 weight_decay_mult=0 bias_learning_rate_mult=1 bias_weight_decay_mult=1 + layer<120> con (num_filters=8, nr=3, nc=3, stride_y=2, stride_x=2, padding_y=0, padding_x=0) learning_rate_mult=1 weight_decay_mult=1 bias_learning_rate_mult=1 bias_weight_decay_mult=0 + layer<121> tag1 + layer<122> relu + layer<123> add_prev1 + layer<124> bn_con eps=1e-05 learning_rate_mult=1 weight_decay_mult=0 bias_learning_rate_mult=1 bias_weight_decay_mult=1 + layer<125> con (num_filters=8, nr=3, nc=3, stride_y=1, stride_x=1, padding_y=1, padding_x=1) learning_rate_mult=1 weight_decay_mult=1 bias_learning_rate_mult=1 bias_weight_decay_mult=0 + layer<126> relu + layer<127> bn_con eps=1e-05 learning_rate_mult=1 weight_decay_mult=0 bias_learning_rate_mult=1 bias_weight_decay_mult=1 + layer<128> con (num_filters=8, nr=3, nc=3, stride_y=1, stride_x=1, padding_y=1, padding_x=1) learning_rate_mult=1 weight_decay_mult=1 bias_learning_rate_mult=1 bias_weight_decay_mult=0 + layer<129> tag1 + layer<130> input<matrix> + */ + + // Now that we know the index numbers for each layer, we can access them + // individually using layer<index>(pnet). For example, to access the output + // tensor for the first prelu layer we can say: + layer<3>(pnet).get_output(); + // Or to print the prelu parameter for layer 7 we can say: + cout << "prelu param: "<< layer<7>(pnet).layer_details().get_initial_param_value() << endl; + + // We can also access layers by their type. This next statement finds the + // first tag1 layer in pnet, and is therefore equivalent to calling + // layer<10>(pnet): + layer<tag1>(pnet); + // The tag layers don't do anything at all and exist simply so you can tag + // parts of your network and access them by layer<tag>(). You can also + // index relative to a tag. So for example, to access the layer immediately + // after tag4 you can say: + layer<tag4,1>(pnet); // Equivalent to layer<38+1>(pnet). + + // Or to access the layer 2 layers after tag4: + layer<tag4,2>(pnet); + // Tagging is a very useful tool for making complex network structures. For + // example, the add_prev1 layer is implemented internally by using a call to + // layer<tag1>(). + + + + // Ok, that's enough talk about defining and inspecting networks. Let's + // talk about training networks! + + // The dnn_trainer will use SGD by default, but you can tell it to use + // different solvers like adam with a weight decay of 0.0005 and the given + // momentum parameters. + dnn_trainer<net_type,adam> trainer(net,adam(0.0005, 0.9, 0.999)); + // Also, if you have multiple graphics cards you can tell the trainer to use + // them together to make the training faster. For example, replacing the + // above constructor call with this one would cause it to use GPU cards 0 + // and 1. + //dnn_trainer<net_type,adam> trainer(net,adam(0.0005, 0.9, 0.999), {0,1}); + + trainer.be_verbose(); + // While the trainer is running it keeps an eye on the training error. If + // it looks like the error hasn't decreased for the last 2000 iterations it + // will automatically reduce the learning rate by 0.1. You can change these + // default parameters to some other values by calling these functions. Or + // disable the automatic shrinking entirely by setting the shrink factor to 1. + trainer.set_iterations_without_progress_threshold(2000); + trainer.set_learning_rate_shrink_factor(0.1); + // The learning rate will start at 1e-3. + trainer.set_learning_rate(1e-3); + trainer.set_synchronization_file("mnist_resnet_sync", std::chrono::seconds(100)); + + + // Now, what if your training dataset is so big it doesn't fit in RAM? You + // make mini-batches yourself, any way you like, and you send them to the + // trainer by repeatedly calling trainer.train_one_step(). + // + // For example, the loop below stream MNIST data to out trainer. + std::vector<matrix<unsigned char>> mini_batch_samples; + std::vector<unsigned long> mini_batch_labels; + dlib::rand rnd(time(0)); + // Loop until the trainer's automatic shrinking has shrunk the learning rate to 1e-6. + // Given our settings, this means it will stop training after it has shrunk the + // learning rate 3 times. + while(trainer.get_learning_rate() >= 1e-6) + { + mini_batch_samples.clear(); + mini_batch_labels.clear(); + + // make a 128 image mini-batch + while(mini_batch_samples.size() < 128) + { + auto idx = rnd.get_random_32bit_number()%training_images.size(); + mini_batch_samples.push_back(training_images[idx]); + mini_batch_labels.push_back(training_labels[idx]); + } + + // Tell the trainer to update the network given this mini-batch + trainer.train_one_step(mini_batch_samples, mini_batch_labels); + + // You can also feed validation data into the trainer by periodically + // calling trainer.test_one_step(samples,labels). Unlike train_one_step(), + // test_one_step() doesn't modify the network, it only computes the testing + // error which it records internally. This testing error will then be print + // in the verbose logging and will also determine when the trainer's + // automatic learning rate shrinking happens. Therefore, test_one_step() + // can be used to perform automatic early stopping based on held out data. + } + + // When you call train_one_step(), the trainer will do its processing in a + // separate thread. This allows the main thread to work on loading data + // while the trainer is busy executing the mini-batches in parallel. + // However, this also means we need to wait for any mini-batches that are + // still executing to stop before we mess with the net object. Calling + // get_net() performs the necessary synchronization. + trainer.get_net(); + + + net.clean(); + serialize("mnist_res_network.dat") << net; + + + // Now we have a trained network. However, it has batch normalization + // layers in it. As is customary, we should replace these with simple + // affine layers before we use the network. This can be accomplished by + // making a network type which is identical to net_type but with the batch + // normalization layers replaced with affine. For example: + using test_net_type = loss_multiclass_log<fc<number_of_classes, + avg_pool_everything< + ares<ares<ares<ares_down< + repeat<9,ares, + ares_down< + ares< + input<matrix<unsigned char>> + >>>>>>>>>>; + // Then we can simply assign our trained net to our testing net. + test_net_type tnet = net; + // Or if you only had a file with your trained network you could deserialize + // it directly into your testing network. + deserialize("mnist_res_network.dat") >> tnet; + + + // And finally, we can run the testing network over our data. + + std::vector<unsigned long> predicted_labels = tnet(training_images); + int num_right = 0; + int num_wrong = 0; + for (size_t i = 0; i < training_images.size(); ++i) + { + if (predicted_labels[i] == training_labels[i]) + ++num_right; + else + ++num_wrong; + + } + cout << "training num_right: " << num_right << endl; + cout << "training num_wrong: " << num_wrong << endl; + cout << "training accuracy: " << num_right/(double)(num_right+num_wrong) << endl; + + predicted_labels = tnet(testing_images); + num_right = 0; + num_wrong = 0; + for (size_t i = 0; i < testing_images.size(); ++i) + { + if (predicted_labels[i] == testing_labels[i]) + ++num_right; + else + ++num_wrong; + + } + cout << "testing num_right: " << num_right << endl; + cout << "testing num_wrong: " << num_wrong << endl; + cout << "testing accuracy: " << num_right/(double)(num_right+num_wrong) << endl; + +} +catch(std::exception& e) +{ + cout << e.what() << endl; +} + |