28 #include "tiny_dnn/util/util.h"
29 #include "tiny_dnn/layers/layer.h"
48 :
Base({vector_type::data}, {vector_type::data}),
51 scale_(float_t(1) / (float_t(1) - dropout_rate_)),
54 mask_.resize(1, std::vector<uint8_t>(in_dim));
61 #ifdef CNN_USE_DEFAULT_MOVE_CONSTRUCTORS
67 void set_dropout_rate(float_t rate)
70 scale_ = float_t(1) / (float_t(1) - dropout_rate_);
89 std::vector<index3d<serial_size_t>>
in_shape()
const override {
93 std::vector<index3d<serial_size_t>>
out_shape()
const override {
98 const std::vector<tensor_t*>& out_data,
99 std::vector<tensor_t*>& out_grad,
100 std::vector<tensor_t*>& in_grad)
override {
101 tensor_t& prev_delta = *in_grad[0];
102 const tensor_t& curr_delta = *out_grad[0];
104 CNN_UNREFERENCED_PARAMETER(in_data);
105 CNN_UNREFERENCED_PARAMETER(out_data);
107 for (serial_size_t sample = 0; sample < static_cast<serial_size_t>(prev_delta.size()); ++sample) {
108 for (serial_size_t i = 0; i < static_cast<serial_size_t>(curr_delta.size()); i++) {
109 prev_delta[sample][i] = mask_[sample][i] * curr_delta[sample][i];
115 std::vector<tensor_t*>& out_data)
override {
116 const tensor_t& in = *in_data[0];
117 tensor_t& out = *out_data[0];
119 const size_t sample_count = in.size();
121 if (mask_.size() < sample_count) {
122 mask_.resize(sample_count, mask_[0]);
125 for (
size_t sample = 0, sample_count = in.size(); sample < sample_count; ++sample) {
127 std::vector<uint8_t>& mask = mask_[sample];
129 const vec_t& in_vec = in[sample];
130 vec_t& out_vec = out[sample];
132 if (phase_ == net_phase::train) {
133 for (
size_t i = 0; i < in_vec.size(); i++)
134 mask[i] = bernoulli(dropout_rate_);
136 for (
size_t i = 0; i < in_vec.size(); i++)
137 out_vec[i] = mask[i] * scale_ * in_vec[i];
140 for (
size_t i = 0, end = in_vec.size(); i < end; i++)
141 out_vec[i] = in_vec[i];
154 std::string
layer_type()
const override {
return "dropout"; }
157 const std::vector<uint8_t>& get_mask(serial_size_t sample_index)
const {
158 return mask_[sample_index];
162 for (
auto& sample : mask_) {
163 std::fill(sample.begin(), sample.end(), 0);
167 template <
class Archive>
168 static void load_and_construct(Archive & ar, cereal::construct<dropout_layer> & construct) {
173 ar(cereal::make_nvp(
"in_size",
in_size), cereal::make_nvp(
"dropout_rate",
dropout_rate), cereal::make_nvp(
"phase", phase));
177 template <
class Archive>
178 void serialize(Archive & ar) {
179 layer::serialize_prolog(ar);
180 ar(cereal::make_nvp(
"in_size", in_size_), cereal::make_nvp(
"dropout_rate", dropout_rate_), cereal::make_nvp(
"phase", phase_));
185 float_t dropout_rate_;
187 serial_size_t in_size_;
188 std::vector<std::vector<uint8_t>> mask_;
Definition: activation_function.h:67
applies dropout to the input
Definition: dropout_layer.h:37
serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition: dropout_layer.h:84
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition: dropout_layer.h:93
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition: dropout_layer.h:97
dropout_layer(serial_size_t in_dim, float_t dropout_rate, net_phase phase=net_phase::train)
Definition: dropout_layer.h:47
float_t dropout_rate() const
number of incoming connections for each output unit
Definition: dropout_layer.h:73
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition: dropout_layer.h:154
serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition: dropout_layer.h:78
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition: dropout_layer.h:114
void set_context(net_phase ctx) override
set dropout-context (training-phase or test-phase)
Definition: dropout_layer.h:149
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition: dropout_layer.h:89
base class of all kind of NN layers
Definition: layer.h:62
serial_size_t in_size() const
!
Definition: layer.h:176