28 #include "tiny_dnn/util/util.h"
29 #include "tiny_dnn/layers/layer.h"
33 template<
typename Activation>
36 CNN_USE_LAYER_MEMBERS;
38 typedef std::vector<std::pair<serial_size_t, serial_size_t> > io_connections;
39 typedef std::vector<std::pair<serial_size_t, serial_size_t> > wi_connections;
40 typedef std::vector<std::pair<serial_size_t, serial_size_t> > wo_connections;
44 serial_size_t out_dim,
47 float_t scale_factor = float_t(1))
48 :
Base(std_input_order(bias_dim > 0)),
49 weight2io_(weight_dim),
54 scale_factor_(scale_factor){}
56 size_t param_size()
const {
57 size_t total_param = 0;
58 for (
auto w : weight2io_)
59 if (w.size() > 0) total_param++;
60 for (
auto b : bias2out_)
61 if (b.size() > 0) total_param++;
66 return max_size(out2wi_);
70 return max_size(in2wo_);
73 void connect_weight(serial_size_t input_index, serial_size_t output_index, serial_size_t weight_index) {
74 weight2io_[weight_index].emplace_back(input_index, output_index);
75 out2wi_[output_index].emplace_back(weight_index, input_index);
76 in2wo_[input_index].emplace_back(weight_index, output_index);
79 void connect_bias(serial_size_t bias_index, serial_size_t output_index) {
80 out2bias_[output_index] = bias_index;
81 bias2out_[bias_index].push_back(output_index);
85 std::vector<tensor_t*>& out_data)
override {
86 const tensor_t& in = *in_data[0];
87 const vec_t& W = (*in_data[1])[0];
88 const vec_t& b = (*in_data[2])[0];
89 tensor_t& a = *out_data[1];
92 for (serial_size_t sample = 0, sample_count =
static_cast<serial_size_t
>(in.size()); sample < sample_count; ++sample) {
93 vec_t& a_sample = a[sample];
96 const wi_connections& connections = out2wi_[i];
98 float_t& a_element = a_sample[i];
100 a_element = float_t(0);
102 for (auto connection : connections)
103 a_element += W[connection.first] * in[sample][connection.second];
105 a_element *= scale_factor_;
106 a_element += b[out2bias_[i]];
110 this->forward_activation(*out_data[0], *out_data[1]);
114 const std::vector<tensor_t*>& out_data,
115 std::vector<tensor_t*>& out_grad,
116 std::vector<tensor_t*>& in_grad)
override {
117 const tensor_t& prev_out = *in_data[0];
118 const vec_t& W = (*in_data[1])[0];
119 vec_t& dW = (*in_grad[1])[0];
120 vec_t& db = (*in_grad[2])[0];
121 tensor_t& prev_delta = *in_grad[0];
122 tensor_t& curr_delta = *out_grad[0];
124 this->backward_activation(*out_grad[0], *out_data[0], curr_delta);
127 for (serial_size_t sample = 0, sample_count =
static_cast<serial_size_t
>(prev_out.size()); sample < sample_count; ++sample) {
129 for (int i = r.begin(); i != r.end(); i++) {
130 const wo_connections& connections = in2wo_[i];
131 float_t delta = float_t(0);
133 for (auto connection : connections)
134 delta += W[connection.first] * curr_delta[sample][connection.second];
136 prev_delta[sample][i] = delta * scale_factor_;
141 for (int i = r.begin(); i < r.end(); i++) {
142 const io_connections& connections = weight2io_[i];
143 float_t diff = float_t(0);
145 for (auto connection : connections)
146 diff += prev_out[sample][connection.first] * curr_delta[sample][connection.second];
148 dW[i] += diff * scale_factor_;
152 for (
size_t i = 0; i < bias2out_.size(); i++) {
153 const std::vector<serial_size_t>& outs = bias2out_[i];
154 float_t diff = float_t(0);
157 diff += curr_delta[sample][o];
165 std::vector<io_connections> weight2io_;
166 std::vector<wi_connections> out2wi_;
167 std::vector<wo_connections> in2wo_;
168 std::vector<std::vector<serial_size_t> > bias2out_;
169 std::vector<size_t> out2bias_;
170 float_t scale_factor_;
single-input, single-output network with activation function
Definition: feedforward_layer.h:37
bool parallelize_
Flag indicating whether the layer/node operations ara paralellized.
Definition: layer.h:696
Definition: partial_connected_layer.h:34
serial_size_t fan_in_size() const override
number of incoming connections for each output unit used only for weight/bias initialization methods ...
Definition: partial_connected_layer.h:65
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition: partial_connected_layer.h:113
serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition: partial_connected_layer.h:69
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition: partial_connected_layer.h:84
Definition: parallel_for.h:70