33 #include "tiny_dnn/core/kernels/conv2d_op.h"
34 #include "tiny_dnn/core/kernels/conv2d_grad_op.h"
35 #include "tiny_dnn/core/kernels/conv2d_op_opencl.h"
36 #include "tiny_dnn/core/kernels/conv2d_op_libdnn.h"
38 #include "tiny_dnn/util/util.h"
39 #include "tiny_dnn/util/image.h"
40 #include "tiny_dnn/activations/activation_function.h"
42 using namespace tiny_dnn::core;
51 template<
typename Activation = activation::
identity>
55 CNN_USE_LAYER_MEMBERS;
74 serial_size_t in_height,
75 serial_size_t window_size,
76 serial_size_t in_channels,
77 serial_size_t out_channels,
78 padding pad_type = padding::valid,
80 serial_size_t w_stride = 1,
81 serial_size_t h_stride = 1,
82 backend_t backend_type = core::default_engine())
85 pad_type, has_bias, w_stride, h_stride,
106 serial_size_t in_height,
107 serial_size_t window_width,
108 serial_size_t window_height,
109 serial_size_t in_channels,
110 serial_size_t out_channels,
111 padding pad_type = padding::valid,
112 bool has_bias =
true,
113 serial_size_t w_stride = 1,
114 serial_size_t h_stride = 1,
115 backend_t backend_type = core::default_engine())
118 pad_type, has_bias, w_stride, h_stride,
139 serial_size_t in_height,
140 serial_size_t window_size,
141 serial_size_t in_channels,
142 serial_size_t out_channels,
144 padding pad_type = padding::valid,
145 bool has_bias =
true,
146 serial_size_t w_stride = 1,
147 serial_size_t h_stride = 1,
148 backend_t backend_type = core::default_engine())
151 pad_type, has_bias, w_stride, h_stride,
173 serial_size_t in_height,
174 serial_size_t window_width,
175 serial_size_t window_height,
176 serial_size_t in_channels,
177 serial_size_t out_channels,
179 padding pad_type = padding::valid,
180 bool has_bias =
true,
181 serial_size_t w_stride = 1,
182 serial_size_t h_stride = 1,
183 backend_t backend_type = core::default_engine())
184 :
Base(std_input_order(has_bias)) {
185 conv_set_params(
shape3d(in_width, in_height, in_channels),
186 window_width, window_height,
187 out_channels, pad_type, has_bias,
190 init_backend(backend_type);
191 Base::set_backend_type(backend_type);
196 :
Base(std::move(other))
197 , params_(std::move(other.params_))
198 , padding_op_(std::move(other.padding_op_))
199 , kernel_fwd_(std::move(other.kernel_fwd_))
200 , kernel_back_(std::move(other.kernel_back_))
201 , cws_(std::move(other.cws_)) {
202 init_backend(std::move(other.engine()));
207 return params_.weight.width_ *
208 params_.weight.height_ * params_.in.depth_;
213 return (params_.weight.width_ / params_.w_stride) *
214 (params_.weight.height_ / params_.h_stride) *
223 std::vector<tensor_t*>& out_data)
override {
225 padding_op_.copy_and_pad_input(*in_data[0], cws_.prev_out_padded_);
227 std::vector<tensor_t*> in_data_(in_data.size());
228 in_data_[0] = in_data_padded(in_data);
230 for (serial_size_t i = 1; i < in_data.size(); ++i) {
231 in_data_[i] = in_data[i];
236 ctx.setParallelize(layer::parallelize());
237 ctx.setEngine(layer::engine());
240 kernel_fwd_->compute(ctx);
244 this->forward_activation(*out_data[0], *out_data[1]);
255 const std::vector<tensor_t*>& out_data,
256 std::vector<tensor_t*>& out_grad,
257 std::vector<tensor_t*>& in_grad)
override {
260 this->backward_activation(*out_grad[0], *out_data[0], *out_grad[1]);
262 std::vector<tensor_t*> in_data_;
263 in_data_.push_back(in_data_padded(in_data));
265 for (serial_size_t i = 1; i < in_data.size(); ++i) {
266 in_data_.push_back(in_data[i]);
269 std::vector<tensor_t*> in_grad_;
270 for (serial_size_t i = 0; i < in_grad.size(); ++i) {
271 in_grad_.push_back(in_grad[i]);
274 if (params_.pad_type == padding::same) {
275 in_grad_[0] = &cws_.prev_delta_padded_;
279 ctx.setParams(¶ms_);
280 ctx.setParallelize(layer::parallelize());
281 ctx.setEngine(layer::engine());
284 kernel_back_->compute(ctx);
287 padding_op_.copy_and_unpad_delta(cws_.prev_delta_padded_, *in_grad[0]);
290 void set_sample_count(serial_size_t sample_count)
override {
291 Base::set_sample_count(sample_count);
292 cws_.prev_delta_padded_.resize(
294 vec_t(params_.in_padded.size(), float_t(0)));
297 std::vector<index3d<serial_size_t>>
in_shape()
const override {
298 if (params_.has_bias) {
299 return { params_.in, params_.weight,
303 return { params_.in, params_.weight };
307 std::vector<index3d<serial_size_t>>
308 out_shape()
const override {
return { params_.out, params_.out }; }
311 return std::string(
"conv");
315 std::string kernel_file()
const override {
316 return std::string(
"../tiny_cnn/core/kernels/cl_kernels/conv_layer_spatial.cl");
320 std::string kernel_header()
const override {
321 std::stringstream ss;
322 ss <<
"#define MULTI\n";
323 ss <<
"#define KERNEL_H " << params_.weight.height_ <<
"\n";
324 ss <<
"#define KERNEL_W " << params_.weight.width_ <<
"\n";
325 ss <<
"#define CHANNELS " << params_.weight.depth_ <<
"\n";
326 ss <<
"#define STRIDE_H " << params_.h_stride <<
"\n";
327 ss <<
"#define STRIDE_W " << params_.w_stride <<
"\n";
328 ss <<
"#define APPLY_BIAS " << params_.has_bias <<
"\n";
329 ss <<
"#define OUTPUT_Z " << params_.out.depth_ <<
"\n";
331 ss <<
"#define ZPAR " << params_.out.depth_ <<
"\n";
335 image<> weight_to_image()
const {
337 const serial_size_t border_width = 1;
338 const auto pitch = params_.weight.width_ + border_width;
339 const auto width = params_.out.depth_ * pitch + border_width;
340 const auto height = params_.in.depth_ * pitch + border_width;
341 const image<>::intensity_t bg_color = 255;
342 const vec_t& W = *this->weights()[0];
344 img.resize(width, height);
347 auto minmax = std::minmax_element(W.begin(), W.end());
349 for (serial_size_t r = 0; r < params_.in.depth_; ++r) {
350 for (serial_size_t c = 0; c < params_.out.depth_; ++c) {
351 if (!params_.tbl.is_connected(c, r))
continue;
353 const auto top = r * pitch + border_width;
354 const auto left = c * pitch + border_width;
356 serial_size_t idx = 0;
358 for (serial_size_t y = 0; y < params_.weight.height_; ++y) {
359 for (serial_size_t x = 0; x < params_.weight.width_; ++x) {
360 idx = c * params_.in.depth_ + r;
361 idx = params_.weight.get_index(x, y, idx);
362 const float_t w = W[idx];
364 img.at(left + x, top + y)
365 =
static_cast<image<>::intensity_t
>(
366 rescale(w, *minmax.first,
367 *minmax.second, 0, 255));
376 template <
class Archive>
377 static void load_and_construct(
378 Archive & ar, cereal::construct<convolutional_layer> & construct) {
379 serial_size_t w_width, w_height, out_ch, w_stride, h_stride;
385 ar(cereal::make_nvp(
"in_size", in),
386 cereal::make_nvp(
"window_width", w_width),
387 cereal::make_nvp(
"window_height", w_height),
388 cereal::make_nvp(
"out_channels", out_ch),
389 cereal::make_nvp(
"connection_table", tbl),
390 cereal::make_nvp(
"pad_type", pad_type),
391 cereal::make_nvp(
"has_bias", has_bias),
392 cereal::make_nvp(
"w_stride", w_stride),
393 cereal::make_nvp(
"h_stride", h_stride)
396 construct(in.width_, in.height_, w_width, w_height, in.depth_,
397 out_ch, tbl, pad_type, has_bias, w_stride, h_stride);
400 template <
class Archive>
401 void serialize(Archive & ar) {
402 layer::serialize_prolog(ar);
403 ar(cereal::make_nvp(
"in_size", params_.in),
404 cereal::make_nvp(
"window_width", params_.weight.width_),
405 cereal::make_nvp(
"window_height", params_.weight.height_),
406 cereal::make_nvp(
"out_channels", params_.out.depth_),
407 cereal::make_nvp(
"connection_table", params_.tbl),
408 cereal::make_nvp(
"pad_type", params_.pad_type),
409 cereal::make_nvp(
"has_bias", params_.has_bias),
410 cereal::make_nvp(
"w_stride", params_.w_stride),
411 cereal::make_nvp(
"h_stride", params_.h_stride)
416 tensor_t* in_data_padded(
const std::vector<tensor_t*>& in) {
417 return (params_.pad_type == padding::valid) ?
418 in[0] : &cws_.prev_out_padded_;
421 void conv_set_params(
const shape3d& in,
422 serial_size_t w_width,
423 serial_size_t w_height,
427 serial_size_t w_stride,
428 serial_size_t h_stride,
431 params_.in_padded = shape3d(in_length(in.width_, w_width, ptype),
432 in_length(in.height_, w_height, ptype),
435 shape3d(conv_out_length(in.width_, w_width, w_stride, ptype),
436 conv_out_length(in.height_, w_height, h_stride, ptype),
438 params_.weight = shape3d(w_width, w_height, in.depth_ * outc);
439 params_.has_bias = has_bias;
440 params_.pad_type = ptype;
441 params_.w_stride = w_stride;
442 params_.h_stride = h_stride;
446 if (params_.pad_type == padding::same) {
447 cws_.prev_delta_padded_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
454 serial_size_t in_length(serial_size_t in_length,
455 serial_size_t window_size, padding pad_type)
const {
456 return pad_type == padding::same ?
457 (in_length + window_size - 1) : in_length;
460 static serial_size_t conv_out_dim(serial_size_t in_width,
461 serial_size_t in_height,
462 serial_size_t window_size,
463 serial_size_t w_stride,
464 serial_size_t h_stride, padding pad_type) {
465 return conv_out_length(in_width, window_size, w_stride, pad_type) *
466 conv_out_length(in_height, window_size, h_stride, pad_type);
469 serial_size_t conv_out_dim(serial_size_t in_width,
470 serial_size_t in_height,
471 serial_size_t window_width,
472 serial_size_t window_height,
473 serial_size_t w_stride,
474 serial_size_t h_stride, padding pad_type)
const {
475 return conv_out_length(in_width, window_width, w_stride, pad_type) *
476 conv_out_length(in_height, window_height, h_stride, pad_type);
479 void createOp()
override {
480 init_backend(layer::engine());
483 void init_backend(
const backend_t backend_type) {
484 core::OpKernelConstruction ctx =
485 core::OpKernelConstruction(layer::device(), ¶ms_);
487 if (backend_type == backend_t::internal ||
488 backend_type == backend_t::nnpack ||
489 backend_type == backend_t::avx) {
491 kernel_fwd_.reset(
new Conv2dOp(ctx));
492 kernel_back_.reset(
new Conv2dGradOp(ctx));
495 else if (backend_type == backend_t::opencl) {
496 throw nn_error(
"Not implemented engine: " + to_string(backend_type));
501 else if (backend_type == backend_t::libdnn) {
502 if (layer::device() ==
nullptr)
return;
503 kernel_fwd_.reset(
new Conv2dLibDNNForwardOp(ctx));
504 kernel_back_.reset(
new Conv2dLibDNNBackwardOp(ctx));
508 throw nn_error(
"Not supported engine: " + to_string(backend_type));
521 std::shared_ptr<core::OpKernel> kernel_fwd_;
522 std::shared_ptr<core::OpKernel> kernel_back_;
526 tensor_t prev_out_padded_;
527 tensor_t prev_delta_padded_;
2D convolution layer
Definition: convolutional_layer.h:52
serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition: convolutional_layer.h:212
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition: convolutional_layer.h:310
serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition: convolutional_layer.h:206
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition: convolutional_layer.h:222
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition: convolutional_layer.h:308
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition: convolutional_layer.h:254
convolutional_layer(convolutional_layer &&other)
number of incoming connections for each output unit
Definition: convolutional_layer.h:195
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition: convolutional_layer.h:73
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition: convolutional_layer.h:172
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition: convolutional_layer.h:138
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition: convolutional_layer.h:105
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition: convolutional_layer.h:297
Definition: conv_params.h:121
Definition: op_kernel.h:72
Definition: conv_params.h:92
single-input, single-output network with activation function
Definition: feedforward_layer.h:37
Definition: conv_params.h:40
Definition: conv_params.h:34