tiny_dnn  1.0.0
A header only, dependency-free deep learning framework in C++11
convolutional_layer.h
1 /*
2  Copyright (c) 2013, Taiga Nomi
3  All rights reserved.
4 
5  Redistribution and use in source and binary forms, with or without
6  modification, are permitted provided that the following conditions are met:
7  * Redistributions of source code must retain the above copyright
8  notice, this list of conditions and the following disclaimer.
9  * Redistributions in binary form must reproduce the above copyright
10  notice, this list of conditions and the following disclaimer in the
11  documentation and/or other materials provided with the distribution.
12  * Neither the name of the <organization> nor the
13  names of its contributors may be used to endorse or promote products
14  derived from this software without specific prior written permission.
15 
16  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
17  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 #pragma once
28 
29 #include <vector>
30 #include <string>
31 #include <algorithm>
32 
33 #include "tiny_dnn/core/kernels/conv2d_op.h"
34 #include "tiny_dnn/core/kernels/conv2d_grad_op.h"
35 #include "tiny_dnn/core/kernels/conv2d_op_opencl.h"
36 #include "tiny_dnn/core/kernels/conv2d_op_libdnn.h"
37 
38 #include "tiny_dnn/util/util.h"
39 #include "tiny_dnn/util/image.h"
40 #include "tiny_dnn/activations/activation_function.h"
41 
42 using namespace tiny_dnn::core;
43 
44 namespace tiny_dnn {
45 
51 template<typename Activation = activation::identity>
52 class convolutional_layer : public feedforward_layer<Activation> {
53  public:
55  CNN_USE_LAYER_MEMBERS;
56 
73  convolutional_layer(serial_size_t in_width,
74  serial_size_t in_height,
75  serial_size_t window_size,
76  serial_size_t in_channels,
77  serial_size_t out_channels,
78  padding pad_type = padding::valid,
79  bool has_bias = true,
80  serial_size_t w_stride = 1,
81  serial_size_t h_stride = 1,
82  backend_t backend_type = core::default_engine())
83  : convolutional_layer(in_width, in_height, window_size, window_size,
84  in_channels, out_channels, connection_table(),
85  pad_type, has_bias, w_stride, h_stride,
86  backend_type) {}
87 
105  convolutional_layer(serial_size_t in_width,
106  serial_size_t in_height,
107  serial_size_t window_width,
108  serial_size_t window_height,
109  serial_size_t in_channels,
110  serial_size_t out_channels,
111  padding pad_type = padding::valid,
112  bool has_bias = true,
113  serial_size_t w_stride = 1,
114  serial_size_t h_stride = 1,
115  backend_t backend_type = core::default_engine())
116  : convolutional_layer(in_width, in_height, window_width, window_height,
117  in_channels, out_channels, connection_table(),
118  pad_type, has_bias, w_stride, h_stride,
119  backend_type) {}
120 
138  convolutional_layer(serial_size_t in_width,
139  serial_size_t in_height,
140  serial_size_t window_size,
141  serial_size_t in_channels,
142  serial_size_t out_channels,
144  padding pad_type = padding::valid,
145  bool has_bias = true,
146  serial_size_t w_stride = 1,
147  serial_size_t h_stride = 1,
148  backend_t backend_type = core::default_engine())
149  : convolutional_layer(in_width, in_height, window_size, window_size,
150  in_channels, out_channels, connection_table,
151  pad_type, has_bias, w_stride, h_stride,
152  backend_type) {}
153 
172  convolutional_layer(serial_size_t in_width,
173  serial_size_t in_height,
174  serial_size_t window_width,
175  serial_size_t window_height,
176  serial_size_t in_channels,
177  serial_size_t out_channels,
179  padding pad_type = padding::valid,
180  bool has_bias = true,
181  serial_size_t w_stride = 1,
182  serial_size_t h_stride = 1,
183  backend_t backend_type = core::default_engine())
184  : Base(std_input_order(has_bias)) {
185  conv_set_params(shape3d(in_width, in_height, in_channels),
186  window_width, window_height,
187  out_channels, pad_type, has_bias,
188  w_stride, h_stride,
190  init_backend(backend_type);
191  Base::set_backend_type(backend_type);
192  }
193 
194  // move constructor
196  : Base(std::move(other))
197  , params_(std::move(other.params_))
198  , padding_op_(std::move(other.padding_op_))
199  , kernel_fwd_(std::move(other.kernel_fwd_))
200  , kernel_back_(std::move(other.kernel_back_))
201  , cws_(std::move(other.cws_)) {
202  init_backend(std::move(other.engine()));
203  }
204 
206  serial_size_t fan_in_size() const override {
207  return params_.weight.width_ *
208  params_.weight.height_ * params_.in.depth_;
209  }
210 
212  serial_size_t fan_out_size() const override {
213  return (params_.weight.width_ / params_.w_stride) *
214  (params_.weight.height_ / params_.h_stride) *
215  params_.out.depth_;
216  }
217 
222  void forward_propagation(const std::vector<tensor_t*>& in_data,
223  std::vector<tensor_t*>& out_data) override {
224  // apply padding to the input tensor
225  padding_op_.copy_and_pad_input(*in_data[0], cws_.prev_out_padded_);
226 
227  std::vector<tensor_t*> in_data_(in_data.size());
228  in_data_[0] = in_data_padded(in_data);
229 
230  for (serial_size_t i = 1; i < in_data.size(); ++i) {
231  in_data_[i] = in_data[i];
232  }
233 
234  // forward convolutional op context
235  auto ctx = OpKernelContext(in_data_, out_data);
236  ctx.setParallelize(layer::parallelize());
237  ctx.setEngine(layer::engine());
238 
239  // launch convolutional kernel
240  kernel_fwd_->compute(ctx);
241 
242  // activations
243  // TODO(edgar/nyanp): refactor and move activations outside
244  this->forward_activation(*out_data[0], *out_data[1]);
245  }
246 
254  void back_propagation(const std::vector<tensor_t*>& in_data,
255  const std::vector<tensor_t*>& out_data,
256  std::vector<tensor_t*>& out_grad,
257  std::vector<tensor_t*>& in_grad) override {
258  // activations
259  // TODO(edgar/nyanp): refactor and move activations outside
260  this->backward_activation(*out_grad[0], *out_data[0], *out_grad[1]);
261 
262  std::vector<tensor_t*> in_data_;
263  in_data_.push_back(in_data_padded(in_data));
264 
265  for (serial_size_t i = 1; i < in_data.size(); ++i) {
266  in_data_.push_back(in_data[i]);
267  }
268 
269  std::vector<tensor_t*> in_grad_;
270  for (serial_size_t i = 0; i < in_grad.size(); ++i) {
271  in_grad_.push_back(in_grad[i]);
272  }
273 
274  if (params_.pad_type == padding::same) {
275  in_grad_[0] = &cws_.prev_delta_padded_;
276  }
277 
278  auto ctx = OpKernelContext(in_data_, out_data, out_grad, in_grad_);
279  ctx.setParams(&params_);
280  ctx.setParallelize(layer::parallelize());
281  ctx.setEngine(layer::engine());
282 
283  // launch convolutional kernel
284  kernel_back_->compute(ctx);
285 
286  // unpad deltas
287  padding_op_.copy_and_unpad_delta(cws_.prev_delta_padded_, *in_grad[0]);
288  }
289 
290  void set_sample_count(serial_size_t sample_count) override {
291  Base::set_sample_count(sample_count);
292  cws_.prev_delta_padded_.resize(
293  sample_count,
294  vec_t(params_.in_padded.size(), float_t(0)));
295  }
296 
297  std::vector<index3d<serial_size_t>> in_shape() const override {
298  if (params_.has_bias) {
299  return { params_.in, params_.weight,
300  index3d<serial_size_t>(1, 1, params_.out.depth_) };
301  }
302  else {
303  return { params_.in, params_.weight };
304  }
305  }
306 
307  std::vector<index3d<serial_size_t>>
308  out_shape() const override { return { params_.out, params_.out }; }
309 
310  std::string layer_type() const override {
311  return std::string("conv");
312  }
313 
314  //TODO(edgar): check this
315  std::string kernel_file() const override {
316  return std::string("../tiny_cnn/core/kernels/cl_kernels/conv_layer_spatial.cl");
317  }
318 
319  //TODO(edgar): is it really needed?
320  std::string kernel_header() const override {
321  std::stringstream ss;
322  ss << "#define MULTI\n";
323  ss << "#define KERNEL_H " << params_.weight.height_ << "\n";
324  ss << "#define KERNEL_W " << params_.weight.width_ << "\n";
325  ss << "#define CHANNELS " << params_.weight.depth_ << "\n";
326  ss << "#define STRIDE_H " << params_.h_stride << "\n";
327  ss << "#define STRIDE_W " << params_.w_stride << "\n";
328  ss << "#define APPLY_BIAS " << params_.has_bias << "\n";
329  ss << "#define OUTPUT_Z " << params_.out.depth_ << "\n";
330  // TODO(edgar): REVISE THIS
331  ss << "#define ZPAR " << params_.out.depth_ << "\n";
332  return ss.str();
333  }
334 
335  image<> weight_to_image() const {
336  image<> img;
337  const serial_size_t border_width = 1;
338  const auto pitch = params_.weight.width_ + border_width;
339  const auto width = params_.out.depth_ * pitch + border_width;
340  const auto height = params_.in.depth_ * pitch + border_width;
341  const image<>::intensity_t bg_color = 255;
342  const vec_t& W = *this->weights()[0];
343 
344  img.resize(width, height);
345  img.fill(bg_color);
346 
347  auto minmax = std::minmax_element(W.begin(), W.end());
348 
349  for (serial_size_t r = 0; r < params_.in.depth_; ++r) {
350  for (serial_size_t c = 0; c < params_.out.depth_; ++c) {
351  if (!params_.tbl.is_connected(c, r)) continue;
352 
353  const auto top = r * pitch + border_width;
354  const auto left = c * pitch + border_width;
355 
356  serial_size_t idx = 0;
357 
358  for (serial_size_t y = 0; y < params_.weight.height_; ++y) {
359  for (serial_size_t x = 0; x < params_.weight.width_; ++x) {
360  idx = c * params_.in.depth_ + r;
361  idx = params_.weight.get_index(x, y, idx);
362  const float_t w = W[idx];
363 
364  img.at(left + x, top + y)
365  = static_cast<image<>::intensity_t>(
366  rescale(w, *minmax.first,
367  *minmax.second, 0, 255));
368  }
369  }
370  }
371  }
372  return img;
373  }
374 
375 
376  template <class Archive>
377  static void load_and_construct(
378  Archive & ar, cereal::construct<convolutional_layer> & construct) {
379  serial_size_t w_width, w_height, out_ch, w_stride, h_stride;
380  bool has_bias;
381  shape3d in;
382  padding pad_type;
383  connection_table tbl;
384 
385  ar(cereal::make_nvp("in_size", in),
386  cereal::make_nvp("window_width", w_width),
387  cereal::make_nvp("window_height", w_height),
388  cereal::make_nvp("out_channels", out_ch),
389  cereal::make_nvp("connection_table", tbl),
390  cereal::make_nvp("pad_type", pad_type),
391  cereal::make_nvp("has_bias", has_bias),
392  cereal::make_nvp("w_stride", w_stride),
393  cereal::make_nvp("h_stride", h_stride)
394  );
395 
396  construct(in.width_, in.height_, w_width, w_height, in.depth_,
397  out_ch, tbl, pad_type, has_bias, w_stride, h_stride);
398  }
399 
400  template <class Archive>
401  void serialize(Archive & ar) {
402  layer::serialize_prolog(ar);
403  ar(cereal::make_nvp("in_size", params_.in),
404  cereal::make_nvp("window_width", params_.weight.width_),
405  cereal::make_nvp("window_height", params_.weight.height_),
406  cereal::make_nvp("out_channels", params_.out.depth_),
407  cereal::make_nvp("connection_table", params_.tbl),
408  cereal::make_nvp("pad_type", params_.pad_type),
409  cereal::make_nvp("has_bias", params_.has_bias),
410  cereal::make_nvp("w_stride", params_.w_stride),
411  cereal::make_nvp("h_stride", params_.h_stride)
412  );
413  }
414 
415 private:
416  tensor_t* in_data_padded(const std::vector<tensor_t*>& in) {
417  return (params_.pad_type == padding::valid) ?
418  in[0] : &cws_.prev_out_padded_;
419  }
420 
421  void conv_set_params(const shape3d& in,
422  serial_size_t w_width,
423  serial_size_t w_height,
424  serial_size_t outc,
425  padding ptype,
426  bool has_bias,
427  serial_size_t w_stride,
428  serial_size_t h_stride,
429  const connection_table& tbl = connection_table()) {
430  params_.in = in;
431  params_.in_padded = shape3d(in_length(in.width_, w_width, ptype),
432  in_length(in.height_, w_height, ptype),
433  in.depth_);
434  params_.out =
435  shape3d(conv_out_length(in.width_, w_width, w_stride, ptype),
436  conv_out_length(in.height_, w_height, h_stride, ptype),
437  outc);
438  params_.weight = shape3d(w_width, w_height, in.depth_ * outc);
439  params_.has_bias = has_bias;
440  params_.pad_type = ptype;
441  params_.w_stride = w_stride;
442  params_.h_stride = h_stride;
443  params_.tbl = tbl;
444 
445  // init padding buffer
446  if (params_.pad_type == padding::same) {
447  cws_.prev_delta_padded_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
448  }
449 
450  // set parameters to padding operation
451  padding_op_ = Conv2dPadding(params_);
452  }
453 
454  serial_size_t in_length(serial_size_t in_length,
455  serial_size_t window_size, padding pad_type) const {
456  return pad_type == padding::same ?
457  (in_length + window_size - 1) : in_length;
458  }
459 
460  static serial_size_t conv_out_dim(serial_size_t in_width,
461  serial_size_t in_height,
462  serial_size_t window_size,
463  serial_size_t w_stride,
464  serial_size_t h_stride, padding pad_type) {
465  return conv_out_length(in_width, window_size, w_stride, pad_type) *
466  conv_out_length(in_height, window_size, h_stride, pad_type);
467  }
468 
469  serial_size_t conv_out_dim(serial_size_t in_width,
470  serial_size_t in_height,
471  serial_size_t window_width,
472  serial_size_t window_height,
473  serial_size_t w_stride,
474  serial_size_t h_stride, padding pad_type) const {
475  return conv_out_length(in_width, window_width, w_stride, pad_type) *
476  conv_out_length(in_height, window_height, h_stride, pad_type);
477  }
478 
479  void createOp() override {
480  init_backend(layer::engine());
481  }
482 
483  void init_backend(const backend_t backend_type) {
484  core::OpKernelConstruction ctx =
485  core::OpKernelConstruction(layer::device(), &params_);
486 
487  if (backend_type == backend_t::internal ||
488  backend_type == backend_t::nnpack ||
489  backend_type == backend_t::avx) {
490 
491  kernel_fwd_.reset(new Conv2dOp(ctx));
492  kernel_back_.reset(new Conv2dGradOp(ctx));
493  return;
494  }
495  else if (backend_type == backend_t::opencl) {
496  throw nn_error("Not implemented engine: " + to_string(backend_type));
497  /*kernel_fwd_.reset(new Conv2dOpenCLForwardOp(ctx));
498  kernel_back_.reset(new Conv2dOpenCLBackwardOp(ctx));
499  return;*/
500  }
501  else if (backend_type == backend_t::libdnn) {
502  if (layer::device() == nullptr) return;
503  kernel_fwd_.reset(new Conv2dLibDNNForwardOp(ctx));
504  kernel_back_.reset(new Conv2dLibDNNBackwardOp(ctx));
505  return;
506  }
507  else {
508  throw nn_error("Not supported engine: " + to_string(backend_type));
509  }
510 
511  }
512 
513  private:
514  /* The convolution parameters */
515  conv_params params_;
516 
517  /* Padding operation */
518  Conv2dPadding padding_op_;
519 
520  /* Forward and backward ops */
521  std::shared_ptr<core::OpKernel> kernel_fwd_;
522  std::shared_ptr<core::OpKernel> kernel_back_;
523 
524  /* Buffer to store padded data */
526  tensor_t prev_out_padded_;
527  tensor_t prev_delta_padded_;
528  } cws_;
529 };
530 
531 } // namespace tiny_dnn
2D convolution layer
Definition: convolutional_layer.h:52
serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition: convolutional_layer.h:212
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition: convolutional_layer.h:310
serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition: convolutional_layer.h:206
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition: convolutional_layer.h:222
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition: convolutional_layer.h:308
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition: convolutional_layer.h:254
convolutional_layer(convolutional_layer &&other)
number of incoming connections for each output unit
Definition: convolutional_layer.h:195
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition: convolutional_layer.h:73
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition: convolutional_layer.h:172
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition: convolutional_layer.h:138
convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing convolutional layer
Definition: convolutional_layer.h:105
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition: convolutional_layer.h:297
Definition: conv_params.h:121
Definition: op_kernel.h:72
Definition: conv_params.h:92
single-input, single-output network with activation function
Definition: feedforward_layer.h:37
Definition: conv_params.h:40