33 #include "tiny_dnn/core/backend_tiny.h"
34 #include "tiny_dnn/core/backend_nnp.h"
35 #include "tiny_dnn/core/backend_dnn.h"
37 #include "tiny_dnn/core/backend_avx.h"
40 #include "tiny_dnn/util/util.h"
41 #include "tiny_dnn/util/image.h"
42 #include "tiny_dnn/activations/activation_function.h"
44 using namespace tiny_dnn::core;
53 template<
typename Activation = activation::
identity>
57 CNN_USE_LAYER_MEMBERS;
75 serial_size_t in_height,
76 serial_size_t window_size,
77 serial_size_t in_channels,
78 serial_size_t out_channels,
79 padding pad_type = padding::valid,
81 serial_size_t w_stride = 1,
82 serial_size_t h_stride = 1,
83 backend_t backend_type = core::backend_t::internal)
84 :
Base(std_input_order(has_bias)) {
85 conv_set_params(
shape3d(in_width, in_height, in_channels),
86 window_size, window_size,
87 out_channels, pad_type, has_bias,
89 init_backend(backend_type);
109 serial_size_t in_height,
110 serial_size_t window_width,
111 serial_size_t window_height,
112 serial_size_t in_channels,
113 serial_size_t out_channels,
114 padding pad_type = padding::valid,
115 bool has_bias =
true,
116 serial_size_t w_stride = 1,
117 serial_size_t h_stride = 1,
118 backend_t backend_type = core::backend_t::internal)
119 :
Base(std_input_order(has_bias)) {
120 conv_set_params(
shape3d(in_width, in_height, in_channels),
121 window_width, window_height,
122 out_channels, pad_type, has_bias,
124 init_backend(backend_type);
144 serial_size_t in_height,
145 serial_size_t window_size,
146 serial_size_t in_channels,
147 serial_size_t out_channels,
149 padding pad_type = padding::valid,
150 bool has_bias =
true,
151 serial_size_t w_stride = 1,
152 serial_size_t h_stride = 1,
153 backend_t backend_type = core::backend_t::internal)
154 :
Base(std_input_order(has_bias)) {
155 conv_set_params(
shape3d(in_width, in_height, in_channels),
156 window_size, window_size,
157 out_channels, pad_type, has_bias,
160 init_backend(backend_type);
181 serial_size_t in_height,
182 serial_size_t window_width,
183 serial_size_t window_height,
184 serial_size_t in_channels,
185 serial_size_t out_channels,
187 padding pad_type = padding::valid,
188 bool has_bias =
true,
189 serial_size_t w_stride = 1,
190 serial_size_t h_stride = 1,
191 backend_t backend_type = core::backend_t::internal)
192 :
Base(has_bias ? 3 : 2, 1, std_input_order(has_bias)) {
193 conv_set_params(
shape3d(in_width, in_height, in_channels),
194 window_width, window_height,
195 out_channels, pad_type, has_bias,
198 init_backend(backend_type);
203 :
Base(std::move(other))
204 , params_(std::move(other.params_))
205 , cws_(std::move(other.cws_)) {
206 init_backend(core::backend_t::internal);
211 return params_.weight.width_ *
212 params_.weight.height_ * params_.in.depth_;
217 return (params_.weight.width_ / params_.w_stride) *
218 (params_.weight.height_ / params_.h_stride) *
227 std::vector<tensor_t*>& out_data)
override {
229 if (in_data.size() == 3) {
230 Base::backend_->conv2d_q(in_data, out_data);
233 this->forward_activation(*out_data[0], *out_data[1]);
234 }
else if (in_data.size() == 6) {
235 Base::backend_->conv2d_eq(in_data, out_data);
247 const std::vector<tensor_t*>& out_data,
248 std::vector<tensor_t*>& out_grad,
249 std::vector<tensor_t*>& in_grad)
override {
250 Base::backend_->conv2d_q(in_data, out_data, out_grad, in_grad);
253 std::vector<index3d<serial_size_t>>
in_shape()
const override {
254 if (params_.has_bias) {
255 return { params_.in, params_.weight,
258 return { params_.in, params_.weight };
262 std::vector<index3d<serial_size_t>>
263 out_shape()
const override {
return { params_.out, params_.out }; }
267 image<> weight_to_image()
const {
269 const serial_size_t border_width = 1;
270 const auto pitch = params_.weight.width_ + border_width;
271 const auto width = params_.out.depth_ * pitch + border_width;
272 const auto height = params_.in.depth_ * pitch + border_width;
273 const image<>::intensity_t bg_color = 255;
274 const vec_t& W = *this->get_weights()[0];
276 img.resize(width, height);
279 auto minmax = std::minmax_element(W.begin(), W.end());
281 for (serial_size_t r = 0; r < params_.in.depth_; ++r) {
282 for (serial_size_t c = 0; c < params_.out.depth_; ++c) {
283 if (!params_.tbl.is_connected(c, r))
continue;
285 const auto top = r * pitch + border_width;
286 const auto left = c * pitch + border_width;
288 serial_size_t idx = 0;
290 for (serial_size_t y = 0; y < params_.weight.height_; ++y) {
291 for (serial_size_t x = 0; x < params_.weight.width_; ++x) {
292 idx = c * params_.in.depth_ + r;
293 idx = params_.weight.get_index(x, y, idx);
294 const float_t w = W[idx];
296 img.at(left + x, top + y)
297 =
static_cast<image<>::intensity_t
>(
298 rescale(w, *minmax.first,
299 *minmax.second, 0, 255));
308 void conv_set_params(
const shape3d& in,
309 serial_size_t w_width,
310 serial_size_t w_height,
314 serial_size_t w_stride,
315 serial_size_t h_stride,
318 params_.in_padded = shape3d(in_length(in.width_, w_width, ptype),
319 in_length(in.height_, w_height, ptype),
322 shape3d(conv_out_length(in.width_, w_width, w_stride, ptype),
323 conv_out_length(in.height_, w_height, h_stride, ptype),
325 params_.weight = shape3d(w_width, w_height, in.depth_ * outc);
326 params_.has_bias = has_bias;
327 params_.pad_type = ptype;
328 params_.w_stride = w_stride;
329 params_.h_stride = h_stride;
334 if (params_.pad_type == padding::same) {
335 cws_.prev_out_buf_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
336 cws_.prev_delta_padded_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
339 cws_.prev_out_buf_.clear();
343 serial_size_t in_length(serial_size_t in_length,
344 serial_size_t window_size, padding pad_type)
const {
345 return pad_type == padding::same ?
346 (in_length + window_size - 1) : in_length;
349 static serial_size_t conv_out_length(serial_size_t in_length,
350 serial_size_t window_size,
351 serial_size_t stride, padding pad_type) {
353 if (pad_type == padding::same) {
354 tmp =
static_cast<float_t
>(in_length) / stride;
355 }
else if (pad_type == padding::valid) {
356 tmp =
static_cast<float_t
>(in_length - window_size + 1) / stride;
358 throw nn_error(
"Not recognized pad_type.");
360 return static_cast<serial_size_t
>(ceil(tmp));
363 static serial_size_t conv_out_dim(serial_size_t in_width,
364 serial_size_t in_height,
365 serial_size_t window_size,
366 serial_size_t w_stride,
367 serial_size_t h_stride, padding pad_type) {
368 return conv_out_length(in_width, window_size, w_stride, pad_type) *
369 conv_out_length(in_height, window_size, h_stride, pad_type);
372 serial_size_t conv_out_dim(serial_size_t in_width,
373 serial_size_t in_height,
374 serial_size_t window_width,
375 serial_size_t window_height,
376 serial_size_t w_stride,
377 serial_size_t h_stride, padding pad_type)
const {
378 return conv_out_length(in_width, window_width, w_stride, pad_type) *
379 conv_out_length(in_height, window_height, h_stride, pad_type);
382 void copy_and_pad_input(
const tensor_t& in) {
385 serial_size_t sample_count =
static_cast<serial_size_t
>(in.size());
387 cws.prev_out_padded_.resize(sample_count);
389 if (params_.pad_type == padding::same) {
390 cws.prev_out_buf_.resize(sample_count, cws.prev_out_buf_[0]);
391 cws.prev_delta_padded_.resize(sample_count, cws.prev_delta_padded_[0]);
394 for (serial_size_t sample = 0; sample < sample_count; ++sample) {
395 if (params_.pad_type == padding::valid) {
396 cws.prev_out_padded_[sample] = &(in[sample]);
399 vec_t* dst = &cws.prev_out_buf_[sample];
402 for (serial_size_t c = 0; c < params_.in.depth_; c++) {
403 float_t *pimg = &(*dst)[params_.in_padded.get_index(params_.weight.width_ / 2, params_.weight.height_ / 2, c)];
404 const float_t *pin = &in[sample][params_.in.get_index(0, 0, c)];
406 for (serial_size_t y = 0; y < params_.in.height_; y++, pin += params_.in.width_, pimg += params_.in_padded.width_) {
407 std::copy(pin, pin + params_.in.width_, pimg);
411 cws.prev_out_padded_[sample] = &(cws.prev_out_buf_[sample]);
416 void copy_and_unpad_delta(
const tensor_t& delta, tensor_t& delta_unpadded) {
417 if (params_.pad_type == padding::valid) {
418 delta_unpadded = delta;
421 for (serial_size_t sample = 0; sample < delta.size(); sample++) {
422 serial_size_t idx = 0;
423 const vec_t& src = delta[sample];
424 vec_t& dst = delta_unpadded[sample];
426 for (serial_size_t c = 0; c < params_.in.depth_; c++) {
427 float_t *pdst = &dst[params_.in.get_index(0, 0, c)];
428 idx = params_.in_padded.get_index(params_.weight.width_ / 2,
429 params_.weight.height_ / 2, c);
430 const float_t *pin = &src[idx];
432 for (serial_size_t y = 0; y < params_.in.height_; y++) {
433 std::copy(pin, pin + params_.in.width_, pdst);
434 pdst += params_.in.width_;
435 pin += params_.in_padded.width_;
442 void init_backend(
const backend_t backend_type) {
443 std::shared_ptr<core::backend>
backend =
nullptr;
446 if (backend_type == backend_t::internal) {
447 backend = std::make_shared<core::tiny_backend>(¶ms_,
448 [
this](
const tensor_t& in) {
449 return copy_and_pad_input(in);
451 [
this](
const tensor_t& delta, tensor_t& dst) {
452 return copy_and_unpad_delta(delta, dst);
454 [
this](
const tensor_t& p_delta,
455 const tensor_t& out, tensor_t& c_delta) {
456 return Base::backward_activation(p_delta, out, c_delta);
459 }
else if (backend_type == backend_t::nnpack) {
460 backend = std::make_shared<core::nnp_backend>(¶ms_,
461 [
this](
const tensor_t& in) {
462 return copy_and_pad_input(in);
465 }
else if (backend_type == backend_t::libdnn) {
466 backend = std::make_shared<core::dnn_backend>();
468 }
else if (backend_type == backend_t::avx) {
469 backend = std::make_shared<core::avx_backend>(¶ms_,
470 [
this](
const tensor_t& in) {
471 return copy_and_pad_input(in);
473 [
this](
const tensor_t& delta, tensor_t& dst) {
474 return copy_and_unpad_delta(delta, dst);
476 [
this](
const tensor_t& p_delta,
477 const tensor_t& out, tensor_t& c_delta) {
478 return Base::backward_activation(p_delta, out, c_delta);
483 throw nn_error(
"Not supported backend type.");
488 Base::backend_->set_layer(
this);
490 throw nn_error(
"Could not allocate the backend.");
Definition: conv_params.h:92
single-input, single-output network with activation function
Definition: feedforward_layer.h:37
Simple image utility class.
Definition: image.h:94
2D convolution layer
Definition: quantized_convolutional_layer.h:54
quantized_convolutional_layer(quantized_convolutional_layer &&other)
number of incoming connections for each output unit
Definition: quantized_convolutional_layer.h:202
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition: quantized_convolutional_layer.h:143
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition: quantized_convolutional_layer.h:246
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition: quantized_convolutional_layer.h:263
serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition: quantized_convolutional_layer.h:216
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition: quantized_convolutional_layer.h:74
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition: quantized_convolutional_layer.h:108
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition: quantized_convolutional_layer.h:265
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition: quantized_convolutional_layer.h:226
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition: quantized_convolutional_layer.h:180
serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition: quantized_convolutional_layer.h:210
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition: quantized_convolutional_layer.h:253
Definition: conv_params.h:40
Definition: conv_params.h:34