tiny_dnn  1.0.0
A header only, dependency-free deep learning framework in C++11
quantized_deconvolutional_layer.h
1 /*
2  Copyright (c) 2013, Taiga Nomi
3  All rights reserved.
4 
5  Redistribution and use in source and binary forms, with or without
6  modification, are permitted provided that the following conditions are met:
7  * Redistributions of source code must retain the above copyright
8  notice, this list of conditions and the following disclaimer.
9  * Redistributions in binary form must reproduce the above copyright
10  notice, this list of conditions and the following disclaimer in the
11  documentation and/or other materials provided with the distribution.
12  * Neither the name of the <organization> nor the
13  names of its contributors may be used to endorse or promote products
14  derived from this software without specific prior written permission.
15 
16  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
17  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 #pragma once
28 
29 #include <deque>
30 #include <string>
31 #include <algorithm>
32 
33 #include "tiny_dnn/core/backend_tiny.h"
34 #include "tiny_dnn/core/backend_nnp.h"
35 #include "tiny_dnn/core/backend_dnn.h"
36 #ifdef CNN_USE_AVX
37 #include "tiny_dnn/core/backend_avx.h"
38 #endif
39 
40 #include "tiny_dnn/util/util.h"
41 #include "tiny_dnn/util/image.h"
42 #include "tiny_dnn/activations/activation_function.h"
43 
44 using namespace tiny_dnn::core;
45 
46 namespace tiny_dnn {
47 
53 template<typename Activation = activation::identity>
55 public:
57  CNN_USE_LAYER_MEMBERS;
58 
74  quantized_deconvolutional_layer(serial_size_t in_width,
75  serial_size_t in_height,
76  serial_size_t window_size,
77  serial_size_t in_channels,
78  serial_size_t out_channels,
79  padding pad_type = padding::valid,
80  bool has_bias = true,
81  serial_size_t w_stride = 1,
82  serial_size_t h_stride = 1,
83  backend_t backend_type = core::backend_t::internal)
84  : Base(std_input_order(has_bias)) {
85  deconv_set_params(shape3d(in_width, in_height, in_channels),
86  window_size, window_size,
87  out_channels, pad_type, has_bias,
88  w_stride, h_stride);
89  init_backend(backend_type);
90  }
91 
108  quantized_deconvolutional_layer(serial_size_t in_width,
109  serial_size_t in_height,
110  serial_size_t window_width,
111  serial_size_t window_height,
112  serial_size_t in_channels,
113  serial_size_t out_channels,
114  padding pad_type = padding::valid,
115  bool has_bias = true,
116  serial_size_t w_stride = 1,
117  serial_size_t h_stride = 1,
118  backend_t backend_type = core::backend_t::internal)
119  : Base(std_input_order(has_bias)) {
120  deconv_set_params(shape3d(in_width, in_height, in_channels),
121  window_width, window_height,
122  out_channels, pad_type, has_bias,
123  w_stride, h_stride);
124  init_backend(backend_type);
125  }
126 
143  quantized_deconvolutional_layer(serial_size_t in_width,
144  serial_size_t in_height,
145  serial_size_t window_size,
146  serial_size_t in_channels,
147  serial_size_t out_channels,
149  padding pad_type = padding::valid,
150  bool has_bias = true,
151  serial_size_t w_stride = 1,
152  serial_size_t h_stride = 1,
153  backend_t backend_type = core::backend_t::internal)
154  : Base(std_input_order(has_bias)) {
155  deconv_set_params(shape3d(in_width, in_height, in_channels),
156  window_size, window_size,
157  out_channels, pad_type, has_bias,
158  w_stride, h_stride,
160  init_backend(backend_type);
161  }
162 
180  quantized_deconvolutional_layer(serial_size_t in_width,
181  serial_size_t in_height,
182  serial_size_t window_width,
183  serial_size_t window_height,
184  serial_size_t in_channels,
185  serial_size_t out_channels,
187  padding pad_type = padding::valid,
188  bool has_bias = true,
189  serial_size_t w_stride = 1,
190  serial_size_t h_stride = 1,
191  backend_t backend_type = core::backend_t::internal)
192  : Base(has_bias ? 3 : 2, 1, std_input_order(has_bias)) {
193  deconv_set_params(shape3d(in_width, in_height, in_channels),
194  window_width, window_height,
195  out_channels, pad_type, has_bias,
196  w_stride, h_stride,
198  init_backend(backend_type);
199  }
200 
201  // move constructor
203  : Base(std::move(other))
204  , params_(std::move(other.params_))
205  , backend_type_(std::move(other.backend_type_))
206  , deconv_layer_worker_storage_(std::move(other.deconv_layer_worker_storage_)) {
207  init_backend(std::move(Base::get_backend_type()));
208  }
209 
211  virtual serial_size_t fan_in_size() const override {
212  return params_.weight.width_ *
213  params_.weight.height_ *
214  params_.in.depth_;
215  }
216 
218  virtual serial_size_t fan_out_size() const override {
219  return (params_.weight.width_ * params_.w_stride) *
220  (params_.weight.height_ * params_.h_stride) *
221  params_.out.depth_;
222  }
223 
224  void forward_propagation(const std::vector<tensor_t*>& in_data,
225  std::vector<tensor_t*>& out_data) override {
226  // launch deconvolutional kernel
227  if (in_data.size() == 3) {
228  Base::backend_->deconv2d_q(in_data, out_data);
229 
230  // activations
231  this->forward_activation(*out_data[0], *out_data[1]);
232  } else if (in_data.size() == 6) {
233  Base::backend_->deconv2d_eq(in_data, out_data);
234  }
235  }
236 
244  void back_propagation(const std::vector<tensor_t*>& in_data,
245  const std::vector<tensor_t*>& out_data,
246  std::vector<tensor_t*>& out_grad,
247  std::vector<tensor_t*>& in_grad) override {
248  Base::backend_->deconv2d_q(in_data, out_data, out_grad, in_grad);
249  }
250 
251  std::vector<index3d<serial_size_t>> in_shape() const override {
252  if (params_.has_bias) {
253  return { params_.in, params_.weight,
254  index3d<serial_size_t>(1, 1, params_.out.depth_) };
255  } else {
256  return { params_.in, params_.weight };
257  }
258  }
259 
260  std::vector<index3d<serial_size_t>> out_shape() const override {
261  return {params_.out_unpadded, params_.out_unpadded};
262  }
263 
264  std::string layer_type() const override { return "q_deconv"; }
265 
266  image<> weightto_image() const {
267  image<> img;
268  const serial_size_t border_width = 1;
269  const auto pitch = params_.weight.width_ + border_width;
270  const auto width = params_.out.depth_ * pitch + border_width;
271  const auto height = params_.in.depth_ * pitch + border_width;
272  const image<>::intensity_t bg_color = 255;
273  const vec_t& W = *this->get_weights()[0];
274 
275  img.resize(width, height);
276  img.fill(bg_color);
277 
278  auto minmax = std::minmax_element(W.begin(), W.end());
279 
280  for (serial_size_t r = 0; r < params_.in.depth_; ++r) {
281  for (serial_size_t c = 0; c < params_.out.depth_; ++c) {
282  if (!params_.tbl.is_connected(c, r)) continue;
283 
284  const auto top = r * pitch + border_width;
285  const auto left = c * pitch + border_width;
286 
287  serial_size_t idx = 0;
288 
289  for (serial_size_t y = 0; y < params_.weight.height_; ++y) {
290  for (serial_size_t x = 0; x < params_.weight.width_; ++x) {
291  idx = params_.weight.get_index(x, y, c * params_.in.depth_ + r);
292  const float_t w = W[idx];
293 
294  img.at(left + x, top + y)
295  = static_cast<image<>::intensity_t>(
296  rescale(w, *minmax.first,
297  *minmax.second, 0, 255));
298  }
299  }
300  }
301  }
302  return img;
303  }
304 
305 private:
306  void init_backend(const backend_t backend_type) {
307  std::shared_ptr<core::backend> backend = nullptr;
308 
309  // allocate new backend
310  if (backend_type == backend_t::internal) {
311  backend = std::make_shared<core::tiny_backend>(&params_,
312  [this](const tensor_t& in) {
313  return copy_and_unpad_output(in);
314  },
315  [this](const tensor_t& delta, tensor_t& dst) {
316  return copy_and_pad_delta(delta, dst);
317  },
318  [this](const tensor_t& p_delta,
319  const tensor_t& out, tensor_t& c_delta) {
320  return Base::backward_activation(p_delta, out, c_delta);
321  },
322  &deconv_layer_worker_storage_);
323  } else if (backend_type == backend_t::nnpack) {
324  backend = std::make_shared<core::nnp_backend>();
325  } else if (backend_type == backend_t::libdnn) {
326  backend = std::make_shared<core::dnn_backend>();
327 #ifdef CNN_USE_AVX
328  } else if (backend_type == backend_t::avx) {
329  backend = std::make_shared<core::avx_backend>(&params_,
330  [this](const tensor_t& in) {
331  return copy_and_unpad_output(in);
332  },
333  [this](const tensor_t& delta, tensor_t& dst) {
334  return copy_and_pad_delta(delta, dst);
335  },
336  [this](const tensor_t& p_delta,
337  const tensor_t& out, tensor_t& c_delta) {
338  return Base::backward_activation(p_delta, out, c_delta);
339  },
340  &deconv_layer_worker_storage_);
341 #endif
342  } else {
343  throw nn_error("Not supported backend type.");
344  }
345 
346  if (backend) {
347  Base::set_backend(backend);
348  Base::backend_->set_layer(this);
349  } else {
350  throw nn_error("Could not allocate the backend.");
351  }
352  }
353 
354  void deconv_set_params(const shape3d& in,
355  serial_size_t w_width,
356  serial_size_t w_height,
357  serial_size_t outc,
358  padding ptype,
359  bool has_bias,
360  serial_size_t w_stride,
361  serial_size_t h_stride,
362  const connection_table& tbl = connection_table()) {
363  params_.in = in;
364  params_.out =
365  shape3d(deconv_out_length(in.width_, w_width, w_stride),
366  deconv_out_length(in.height_, w_height, h_stride),
367  outc);
368  params_.out_unpadded =
369  shape3d(deconv_out_unpadded_length(in.width_, w_width, w_stride, ptype),
370  deconv_out_unpadded_length(in.height_, w_height, h_stride, ptype),
371  outc);
372  params_.weight = shape3d(w_width, w_height, in.depth_ * outc);
373  params_.has_bias = has_bias;
374  params_.pad_type = ptype;
375  params_.w_stride = w_stride;
376  params_.h_stride = h_stride;
377  params_.tbl = tbl;
378  }
379 
380  void init() {
381  deconv_layer_worker_specific_storage& dws = deconv_layer_worker_storage_;
382 
383  if (params_.pad_type == padding::same) {
384  dws.curr_out_buf_.resize(1, vec_t(params_.in.size(), float_t(0)));
385  dws.curr_delta_padded.resize(1, vec_t(params_.out.size(), float_t(0)));
386  }
387  else {
388  dws.curr_out_buf_.clear();
389  }
390  }
391 
392  serial_size_t in_length(serial_size_t in_length,
393  serial_size_t window_size, padding pad_type) const {
394  return in_length;
395  }
396 
397  static serial_size_t deconv_out_length(serial_size_t in_length,
398  serial_size_t window_size,
399  serial_size_t stride) {
400  return (serial_size_t)ceil((float_t)(in_length) * stride + window_size - 1);
401  }
402 
403  static serial_size_t deconv_out_unpadded_length(serial_size_t in_length,
404  serial_size_t window_size,
405  serial_size_t stride, padding pad_type) {
406  return pad_type == padding::same ?
407  (serial_size_t)ceil((float_t)in_length * stride) :
408  (serial_size_t)ceil((float_t)(in_length) * stride + window_size - 1);
409  }
410 
411  static serial_size_t deconv_out_dim(serial_size_t in_width,
412  serial_size_t in_height,
413  serial_size_t window_size,
414  serial_size_t w_stride,
415  serial_size_t h_stride, padding pad_type) {
416  return deconv_out_unpadded_length(in_width, window_size, w_stride, pad_type) *
417  deconv_out_unpadded_length(in_height, window_size, h_stride, pad_type);
418  }
419 
420  serial_size_t deconv_out_dim(serial_size_t in_width,
421  serial_size_t in_height,
422  serial_size_t window_width,
423  serial_size_t window_height,
424  serial_size_t w_stride,
425  serial_size_t h_stride, padding pad_type) const {
426  return deconv_out_unpadded_length(in_width, window_width, w_stride, pad_type) *
427  deconv_out_unpadded_length(in_height, window_height, h_stride, pad_type);
428  }
429 
430  void copy_and_pad_delta(const tensor_t& delta, tensor_t& delta_padded) {
431  if (params_.pad_type == padding::valid) {
432  delta_padded = delta;
433  }
434  else {
435  for (serial_size_t sample = 0; sample < delta.size(); sample++) {
436  vec_t& dst = delta_padded[sample];
437  const vec_t& src = delta[sample];
438 
439  for (serial_size_t c = 0; c < params_.in.depth_; c++) {
440  float_t *pdst = &dst[params_.in.get_index(0, 0, c)];
441  const float_t *pin = &src[params_.in.get_index(0, 0, c)];
442 
443  for (serial_size_t y = 0; y < params_.in.height_; y++, pdst +=
444  params_.in.width_, pin += params_.in.width_) {
445  std::copy(pin, pin + params_.in.width_, pdst);
446  }
447  }
448  }
449  }
450  }
451 
452  void copy_and_unpad_output(const tensor_t& out) {
454  deconv_layer_worker_storage_;
455 
456  dws.curr_out_buf_ = tensor_t(out.size(), vec_t(params_.out_unpadded.width_*
457  params_.out_unpadded.height_*
458  params_.out_unpadded.depth_,0));
459  tensor_t* dst_tensor = &dws.curr_out_buf_;
460 
461  if (params_.pad_type == padding::valid) {
462  dws.curr_out_unpadded_ = &out;
463  } else {
464  // make unpadded version in order to restore scale in fprop/bprop
465  for (serial_size_t sample = 0; sample < out.size(); sample++) {
466  serial_size_t idx = 0;
467  vec_t& dst = (*dst_tensor)[sample];
468 
469  for (serial_size_t c = 0; c < params_.out_unpadded.depth_; c++) {
470  float_t *pimg = &dst[params_.out_unpadded.get_index(0, 0, c)];
471  idx = params_.out.get_index(static_cast<serial_size_t>(floor(params_.weight.width_ / 2)),
472  static_cast<serial_size_t>(floor(params_.weight.height_ / 2)), c);
473 
474  const float_t *pout = &out[sample][idx];
475 
476  for (serial_size_t y = static_cast<serial_size_t>(floor(params_.weight.height_ / 2));
477  y < params_.out_unpadded.height_ + static_cast<serial_size_t>(floor(params_.weight.height_ / 2));
478  y++,
479  pout += params_.out.width_,
480  pimg += params_.out_unpadded.width_) {
481  std::copy(pout,
482  pout + params_.out_unpadded.width_,
483  pimg);
484  }
485  }
486  dws.curr_out_unpadded_ = &dws.curr_out_buf_;
487  }
488  }
489  }
490 
491  /* The convolution parameters */
492  deconv_params params_;
493 
494  /* The type of backend */
495  backend_t backend_type_;
496 
497  /* Workers buffers */
498  deconv_layer_worker_specific_storage deconv_layer_worker_storage_;
499 };
500 
501 } // namespace tiny_dnn
Definition: backend.h:68
single-input, single-output network with activation function
Definition: feedforward_layer.h:37
Simple image utility class.
Definition: image.h:94
2D deconvolution layer
Definition: quantized_deconvolutional_layer.h:54
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition: quantized_deconvolutional_layer.h:251
quantized_deconvolutional_layer(quantized_deconvolutional_layer &&other)
number of incoming connections for each output unit
Definition: quantized_deconvolutional_layer.h:202
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition: quantized_deconvolutional_layer.h:260
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition: quantized_deconvolutional_layer.h:224
virtual serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition: quantized_deconvolutional_layer.h:211
quantized_deconvolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing deconvolutional layer
Definition: quantized_deconvolutional_layer.h:74
virtual serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition: quantized_deconvolutional_layer.h:218
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition: quantized_deconvolutional_layer.h:244
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition: quantized_deconvolutional_layer.h:264
quantized_deconvolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing deconvolutional layer
Definition: quantized_deconvolutional_layer.h:180
quantized_deconvolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing deconvolutional layer
Definition: quantized_deconvolutional_layer.h:108
quantized_deconvolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing deconvolutional layer
Definition: quantized_deconvolutional_layer.h:143
Definition: conv_params.h:40
Definition: deconv_params.h:39