tiny_dnn  1.0.0
A header only, dependency-free deep learning framework in C++11
deconvolutional_layer.h
1 /*
2  Copyright (c) 2013, Taiga Nomi
3  All rights reserved.
4 
5  Redistribution and use in source and binary forms, with or without
6  modification, are permitted provided that the following conditions are met:
7  * Redistributions of source code must retain the above copyright
8  notice, this list of conditions and the following disclaimer.
9  * Redistributions in binary form must reproduce the above copyright
10  notice, this list of conditions and the following disclaimer in the
11  documentation and/or other materials provided with the distribution.
12  * Neither the name of the <organization> nor the
13  names of its contributors may be used to endorse or promote products
14  derived from this software without specific prior written permission.
15 
16  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
17  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 #pragma once
28 
29 #include <deque>
30 #include <string>
31 #include <algorithm>
32 
33 #include "tiny_dnn/core/backend_tiny.h"
34 #include "tiny_dnn/core/backend_nnp.h"
35 #include "tiny_dnn/core/backend_dnn.h"
36 #ifdef CNN_USE_AVX
37 #include "tiny_dnn/core/backend_avx.h"
38 #endif
39 
40 #include "tiny_dnn/util/util.h"
41 #include "tiny_dnn/util/image.h"
42 #include "tiny_dnn/activations/activation_function.h"
43 
44 using namespace tiny_dnn::core;
45 
46 namespace tiny_dnn {
47 
53 template<typename Activation = activation::identity>
54 class deconvolutional_layer : public feedforward_layer<Activation> {
55 public:
57  CNN_USE_LAYER_MEMBERS;
58 
74  deconvolutional_layer(serial_size_t in_width,
75  serial_size_t in_height,
76  serial_size_t window_size,
77  serial_size_t in_channels,
78  serial_size_t out_channels,
79  padding pad_type = padding::valid,
80  bool has_bias = true,
81  serial_size_t w_stride = 1,
82  serial_size_t h_stride = 1,
83  backend_t backend_type = core::default_engine())
84  : Base(std_input_order(has_bias)) {
85  deconv_set_params(shape3d(in_width, in_height, in_channels),
86  window_size, window_size,
87  out_channels, pad_type, has_bias,
88  w_stride, h_stride);
89  init_backend(backend_type);
90  }
91 
108  deconvolutional_layer(serial_size_t in_width,
109  serial_size_t in_height,
110  serial_size_t window_width,
111  serial_size_t window_height,
112  serial_size_t in_channels,
113  serial_size_t out_channels,
114  padding pad_type = padding::valid,
115  bool has_bias = true,
116  serial_size_t w_stride = 1,
117  serial_size_t h_stride = 1,
118  backend_t backend_type = core::default_engine())
119  : Base(std_input_order(has_bias)) {
120  deconv_set_params(shape3d(in_width, in_height, in_channels),
121  window_width, window_height,
122  out_channels, pad_type, has_bias,
123  w_stride, h_stride);
124  init_backend(backend_type);
125  }
126 
143  deconvolutional_layer(serial_size_t in_width,
144  serial_size_t in_height,
145  serial_size_t window_size,
146  serial_size_t in_channels,
147  serial_size_t out_channels,
149  padding pad_type = padding::valid,
150  bool has_bias = true,
151  serial_size_t w_stride = 1,
152  serial_size_t h_stride = 1,
153  backend_t backend_type = core::default_engine())
154  : Base(std_input_order(has_bias)) {
155  deconv_set_params(shape3d(in_width, in_height, in_channels),
156  window_size, window_size,
157  out_channels, pad_type, has_bias,
158  w_stride, h_stride,
160  init_backend(backend_type);
161  }
162 
180  deconvolutional_layer(serial_size_t in_width,
181  serial_size_t in_height,
182  serial_size_t window_width,
183  serial_size_t window_height,
184  serial_size_t in_channels,
185  serial_size_t out_channels,
187  padding pad_type = padding::valid,
188  bool has_bias = true,
189  serial_size_t w_stride = 1,
190  serial_size_t h_stride = 1,
191  backend_t backend_type = core::default_engine())
192  : Base(has_bias ? 3 : 2, 1, std_input_order(has_bias)) {
193  deconv_set_params(shape3d(in_width, in_height, in_channels),
194  window_width, window_height,
195  out_channels, pad_type, has_bias,
196  w_stride, h_stride,
198  init_backend(backend_type);
199  }
200 
201  // move constructor
203  : Base(std::move(other))
204  , params_(std::move(other.params_))
205  , backend_type_(std::move(other.backend_type_))
206  , deconv_layer_worker_storage_(std::move(other.deconv_layer_worker_storage_)) {
207  init_backend(std::move(Base::backend_type()));
208  }
209 
211  virtual serial_size_t fan_in_size() const override {
212  return params_.weight.width_ *
213  params_.weight.height_ *
214  params_.in.depth_;
215  }
216 
218  virtual serial_size_t fan_out_size() const override {
219  return (params_.weight.width_ * params_.w_stride) *
220  (params_.weight.height_ * params_.h_stride) *
221  params_.out.depth_;
222  }
223 
224  void forward_propagation(const std::vector<tensor_t*>& in_data,
225  std::vector<tensor_t*>& out_data) override {
226  // launch deconvolutional kernel
227  Base::backend_->deconv2d(in_data, out_data);
228 
229  // activations
230  this->forward_activation(*out_data[0], *out_data[1]);
231  }
232 
241  void back_propagation(const std::vector<tensor_t*>& in_data,
242  const std::vector<tensor_t*>& out_data,
243  std::vector<tensor_t*>& out_grad,
244  std::vector<tensor_t*>& in_grad) override {
245  Base::backend_->deconv2d(in_data, out_data, out_grad, in_grad);
246  }
247 
248  std::vector<index3d<serial_size_t>> in_shape() const override {
249  if (params_.has_bias) {
250  return { params_.in, params_.weight,
251  index3d<serial_size_t>(1, 1, params_.out.depth_) };
252  } else {
253  return { params_.in, params_.weight };
254  }
255  }
256 
257  std::vector<index3d<serial_size_t>> out_shape() const override {
258  return {params_.out_unpadded, params_.out_unpadded};
259  }
260 
261  std::string layer_type() const override { return "deconv"; }
262 
263  image<> weightto_image() const {
264  image<> img;
265  const serial_size_t border_width = 1;
266  const auto pitch = params_.weight.width_ + border_width;
267  const auto width = params_.out.depth_ * pitch + border_width;
268  const auto height = params_.in.depth_ * pitch + border_width;
269  const image<>::intensity_t bg_color = 255;
270  const vec_t& W = *this->get_weights()[0];
271 
272  img.resize(width, height);
273  img.fill(bg_color);
274 
275  auto minmax = std::minmax_element(W.begin(), W.end());
276 
277  for (serial_size_t r = 0; r < params_.in.depth_; ++r) {
278  for (serial_size_t c = 0; c < params_.out.depth_; ++c) {
279  if (!params_.tbl.is_connected(c, r)) continue;
280 
281  const auto top = r * pitch + border_width;
282  const auto left = c * pitch + border_width;
283 
284  serial_size_t idx = 0;
285 
286  for (serial_size_t y = 0; y < params_.weight.height_; ++y) {
287  for (serial_size_t x = 0; x < params_.weight.width_; ++x) {
288  idx = params_.weight.get_index(x, y, c * params_.in.depth_ + r);
289  const float_t w = W[idx];
290 
291  img.at(left + x, top + y)
292  = static_cast<image<>::intensity_t>(
293  rescale(w, *minmax.first,
294  *minmax.second, 0, 255));
295  }
296  }
297  }
298  }
299  return img;
300  }
301 
302 private:
303  void init_backend(const backend_t backend_type) {
304  std::shared_ptr<core::backend> backend = nullptr;
305 
306  // allocate new backend
307  if (backend_type == backend_t::internal) {
308  backend = std::make_shared<core::tiny_backend>(&params_,
309  [this](const tensor_t& in) {
310  return copy_and_unpad_output(in);
311  },
312  [this](const tensor_t& delta, tensor_t& dst) {
313  return copy_and_pad_delta(delta, dst);
314  },
315  [this](const tensor_t& p_delta,
316  const tensor_t& out, tensor_t& c_delta) {
317  return Base::backward_activation(p_delta, out, c_delta);
318  },
319  &deconv_layer_worker_storage_);
320  } else if (backend_type == backend_t::nnpack) {
321  backend = std::make_shared<core::nnp_backend>();
322  } else if (backend_type == backend_t::libdnn) {
323  backend = std::make_shared<core::dnn_backend>();
324 #ifdef CNN_USE_AVX
325  } else if (backend_type == backend_t::avx) {
326  backend = std::make_shared<core::avx_backend>(&params_,
327  [this](const tensor_t& in) {
328  return copy_and_unpad_output(in);
329  },
330  [this](const tensor_t& delta, tensor_t& dst) {
331  return copy_and_pad_delta(delta, dst);
332  },
333  [this](const tensor_t& p_delta,
334  const tensor_t& out, tensor_t& c_delta) {
335  return Base::backward_activation(p_delta, out, c_delta);
336  },
337  &deconv_layer_worker_storage_);
338 #endif
339  } else {
340  throw nn_error("Not supported backend type.");
341  }
342 
343  if (backend) {
344  Base::set_backend(backend);
345  Base::backend_->set_layer(this);
346  } else {
347  throw nn_error("Could not allocate the backend.");
348  }
349  }
350 
351  void deconv_set_params(const shape3d& in,
352  serial_size_t w_width,
353  serial_size_t w_height,
354  serial_size_t outc,
355  padding ptype,
356  bool has_bias,
357  serial_size_t w_stride,
358  serial_size_t h_stride,
359  const connection_table& tbl = connection_table()) {
360  params_.in = in;
361  params_.out =
362  shape3d(deconv_out_length(in.width_, w_width, w_stride),
363  deconv_out_length(in.height_, w_height, h_stride),
364  outc);
365  params_.out_unpadded =
366  shape3d(deconv_out_unpadded_length(in.width_, w_width, w_stride, ptype),
367  deconv_out_unpadded_length(in.height_, w_height, h_stride, ptype),
368  outc);
369  params_.weight = shape3d(w_width, w_height, in.depth_ * outc);
370  params_.has_bias = has_bias;
371  params_.pad_type = ptype;
372  params_.w_stride = w_stride;
373  params_.h_stride = h_stride;
374  params_.tbl = tbl;
375  }
376 
377  void init_workers(serial_size_t sample_count) {
378  deconv_layer_worker_specific_storage& dws = deconv_layer_worker_storage_;
379 
380  if (params_.pad_type == padding::same) {
381  dws.curr_out_buf_.resize(sample_count, vec_t(params_.out_unpadded.size(), float_t(0)));
382  dws.curr_delta_padded.resize(sample_count, vec_t(params_.out.size(), float_t(0)));
383  }
384  else {
385  dws.curr_out_buf_.clear();
386  }
387 
388  }
389 
390  serial_size_t in_length(serial_size_t in_length,
391  serial_size_t window_size, padding pad_type) const {
392  return in_length;
393  }
394 
395  static serial_size_t deconv_out_length(serial_size_t in_length,
396  serial_size_t window_size,
397  serial_size_t stride) {
398  return (serial_size_t)ceil((float_t)(in_length) * stride + window_size - 1);
399  }
400 
401  static serial_size_t deconv_out_unpadded_length(serial_size_t in_length,
402  serial_size_t window_size,
403  serial_size_t stride, padding pad_type) {
404  return pad_type == padding::same ?
405  (serial_size_t)ceil((float_t)in_length * stride) :
406  (serial_size_t)ceil((float_t)(in_length) * stride + window_size - 1);
407  }
408 
409  static serial_size_t deconv_out_dim(serial_size_t in_width,
410  serial_size_t in_height,
411  serial_size_t window_size,
412  serial_size_t w_stride,
413  serial_size_t h_stride, padding pad_type) {
414  return deconv_out_unpadded_length(in_width, window_size, w_stride, pad_type) *
415  deconv_out_unpadded_length(in_height, window_size, h_stride, pad_type);
416  }
417 
418  serial_size_t deconv_out_dim(serial_size_t in_width,
419  serial_size_t in_height,
420  serial_size_t window_width,
421  serial_size_t window_height,
422  serial_size_t w_stride,
423  serial_size_t h_stride, padding pad_type) const {
424  return deconv_out_unpadded_length(in_width, window_width, w_stride, pad_type) *
425  deconv_out_unpadded_length(in_height, window_height, h_stride, pad_type);
426  }
427 
428  void copy_and_pad_delta(const tensor_t& delta, tensor_t& delta_padded) {
429  if (params_.pad_type == padding::valid) {
430  delta_padded = delta;
431  }
432  else {
433  for (serial_size_t sample = 0; sample < delta.size(); sample++) {
434  vec_t& dst = delta_padded[sample];
435  const vec_t& src = delta[sample];
436 
437  for (serial_size_t c = 0; c < params_.in.depth_; c++) {
438  float_t *pdst = &dst[params_.in.get_index(0, 0, c)];
439  const float_t *pin = &src[params_.in.get_index(0, 0, c)];
440 
441  for (serial_size_t y = 0; y < params_.in.height_; y++, pdst +=
442  params_.in.width_, pin += params_.in.width_) {
443  std::copy(pin, pin + params_.in.width_, pdst);
444  }
445  }
446  }
447  }
448  }
449 
450  void copy_and_unpad_output(const tensor_t& out) {
452  deconv_layer_worker_storage_;
453 
454  dws.curr_out_buf_ = tensor_t(out.size(), vec_t(params_.out_unpadded.width_*
455  params_.out_unpadded.height_*
456  params_.out_unpadded.depth_,0));
457  tensor_t* dst_tensor = &dws.curr_out_buf_;
458 
459  if (params_.pad_type == padding::valid) {
460  dws.curr_out_unpadded_ = &out;
461  } else {
462  // make unpadded version in order to restore scale in fprop/bprop
463  for (serial_size_t sample = 0; sample < out.size(); sample++) {
464  serial_size_t idx = 0;
465  vec_t& dst = (*dst_tensor)[sample];
466  serial_size_t wieght_w_half = params_.weight.width_ / 2;
467  serial_size_t wieght_h_half = params_.weight.height_ / 2;
468 
469  for (serial_size_t c = 0; c < params_.out_unpadded.depth_; c++) {
470  float_t *pimg = &dst[params_.out_unpadded.get_index(0, 0, c)];
471  idx = params_.out.get_index(wieght_w_half, wieght_h_half, c);
472  const float_t *pout = &out[sample][idx];
473 
474  for (serial_size_t y = wieght_h_half;
475  y < params_.out_unpadded.height_ + wieght_h_half;
476  y++,
477  pout += params_.out.width_,
478  pimg += params_.out_unpadded.width_) {
479  std::copy(pout,
480  pout + params_.out_unpadded.width_,
481  pimg);
482  }
483  }
484  }
485 
486  dws.curr_out_unpadded_ = &dws.curr_out_buf_;
487  }
488  }
489 
490  /* The convolution parameters */
491  deconv_params params_;
492 
493  /* The type of backend */
494  backend_t backend_type_;
495 
496  deconv_layer_worker_specific_storage deconv_layer_worker_storage_;
497 };
498 
499 } // namespace tiny_dnn
Definition: backend.h:68
2D deconvolution layer
Definition: deconvolutional_layer.h:54
deconvolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing deconvolutional layer
Definition: deconvolutional_layer.h:180
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition: deconvolutional_layer.h:261
deconvolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing deconvolutional layer
Definition: deconvolutional_layer.h:108
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition: deconvolutional_layer.h:248
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition: deconvolutional_layer.h:224
deconvolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing deconvolutional layer
Definition: deconvolutional_layer.h:143
virtual serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition: deconvolutional_layer.h:211
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition: deconvolutional_layer.h:241
virtual serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition: deconvolutional_layer.h:218
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition: deconvolutional_layer.h:257
deconvolutional_layer(deconvolutional_layer &&other)
number of incoming connections for each output unit
Definition: deconvolutional_layer.h:202
deconvolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::default_engine())
constructing deconvolutional layer
Definition: deconvolutional_layer.h:74
single-input, single-output network with activation function
Definition: feedforward_layer.h:37
Simple image utility class.
Definition: image.h:94
Definition: conv_params.h:40
Definition: deconv_params.h:39