tiny_dnn  1.0.0
A header only, dependency-free deep learning framework in C++11
quantized_convolutional_layer.h
1 /*
2  Copyright (c) 2013, Taiga Nomi
3  All rights reserved.
4 
5  Redistribution and use in source and binary forms, with or without
6  modification, are permitted provided that the following conditions are met:
7  * Redistributions of source code must retain the above copyright
8  notice, this list of conditions and the following disclaimer.
9  * Redistributions in binary form must reproduce the above copyright
10  notice, this list of conditions and the following disclaimer in the
11  documentation and/or other materials provided with the distribution.
12  * Neither the name of the <organization> nor the
13  names of its contributors may be used to endorse or promote products
14  derived from this software without specific prior written permission.
15 
16  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
17  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 #pragma once
28 
29 #include <vector>
30 #include <string>
31 #include <algorithm>
32 
33 #include "tiny_dnn/core/backend_tiny.h"
34 #include "tiny_dnn/core/backend_nnp.h"
35 #include "tiny_dnn/core/backend_dnn.h"
36 #ifdef CNN_USE_AVX
37 #include "tiny_dnn/core/backend_avx.h"
38 #endif
39 
40 #include "tiny_dnn/util/util.h"
41 #include "tiny_dnn/util/image.h"
42 #include "tiny_dnn/activations/activation_function.h"
43 
44 using namespace tiny_dnn::core;
45 
46 namespace tiny_dnn {
47 
53 template<typename Activation = activation::identity>
55  public:
57  CNN_USE_LAYER_MEMBERS;
58 
74  quantized_convolutional_layer(serial_size_t in_width,
75  serial_size_t in_height,
76  serial_size_t window_size,
77  serial_size_t in_channels,
78  serial_size_t out_channels,
79  padding pad_type = padding::valid,
80  bool has_bias = true,
81  serial_size_t w_stride = 1,
82  serial_size_t h_stride = 1,
83  backend_t backend_type = core::backend_t::internal)
84  : Base(std_input_order(has_bias)) {
85  conv_set_params(shape3d(in_width, in_height, in_channels),
86  window_size, window_size,
87  out_channels, pad_type, has_bias,
88  w_stride, h_stride);
89  init_backend(backend_type);
90  }
91 
108  quantized_convolutional_layer(serial_size_t in_width,
109  serial_size_t in_height,
110  serial_size_t window_width,
111  serial_size_t window_height,
112  serial_size_t in_channels,
113  serial_size_t out_channels,
114  padding pad_type = padding::valid,
115  bool has_bias = true,
116  serial_size_t w_stride = 1,
117  serial_size_t h_stride = 1,
118  backend_t backend_type = core::backend_t::internal)
119  : Base(std_input_order(has_bias)) {
120  conv_set_params(shape3d(in_width, in_height, in_channels),
121  window_width, window_height,
122  out_channels, pad_type, has_bias,
123  w_stride, h_stride);
124  init_backend(backend_type);
125  }
126 
143  quantized_convolutional_layer(serial_size_t in_width,
144  serial_size_t in_height,
145  serial_size_t window_size,
146  serial_size_t in_channels,
147  serial_size_t out_channels,
149  padding pad_type = padding::valid,
150  bool has_bias = true,
151  serial_size_t w_stride = 1,
152  serial_size_t h_stride = 1,
153  backend_t backend_type = core::backend_t::internal)
154  : Base(std_input_order(has_bias)) {
155  conv_set_params(shape3d(in_width, in_height, in_channels),
156  window_size, window_size,
157  out_channels, pad_type, has_bias,
158  w_stride, h_stride,
160  init_backend(backend_type);
161  }
162 
180  quantized_convolutional_layer(serial_size_t in_width,
181  serial_size_t in_height,
182  serial_size_t window_width,
183  serial_size_t window_height,
184  serial_size_t in_channels,
185  serial_size_t out_channels,
187  padding pad_type = padding::valid,
188  bool has_bias = true,
189  serial_size_t w_stride = 1,
190  serial_size_t h_stride = 1,
191  backend_t backend_type = core::backend_t::internal)
192  : Base(has_bias ? 3 : 2, 1, std_input_order(has_bias)) {
193  conv_set_params(shape3d(in_width, in_height, in_channels),
194  window_width, window_height,
195  out_channels, pad_type, has_bias,
196  w_stride, h_stride,
198  init_backend(backend_type);
199  }
200 
201  // move constructor
203  : Base(std::move(other))
204  , params_(std::move(other.params_))
205  , cws_(std::move(other.cws_)) {
206  init_backend(core::backend_t::internal);
207  }
208 
210  serial_size_t fan_in_size() const override {
211  return params_.weight.width_ *
212  params_.weight.height_ * params_.in.depth_;
213  }
214 
216  serial_size_t fan_out_size() const override {
217  return (params_.weight.width_ / params_.w_stride) *
218  (params_.weight.height_ / params_.h_stride) *
219  params_.out.depth_;
220  }
221 
226  void forward_propagation(const std::vector<tensor_t*>& in_data,
227  std::vector<tensor_t*>& out_data) override {
228  // launch convolutional kernel
229  if (in_data.size() == 3) {
230  Base::backend_->conv2d_q(in_data, out_data);
231 
232  // activations
233  this->forward_activation(*out_data[0], *out_data[1]);
234  } else if (in_data.size() == 6) {
235  Base::backend_->conv2d_eq(in_data, out_data);
236  }
237  }
238 
246  void back_propagation(const std::vector<tensor_t*>& in_data,
247  const std::vector<tensor_t*>& out_data,
248  std::vector<tensor_t*>& out_grad,
249  std::vector<tensor_t*>& in_grad) override {
250  Base::backend_->conv2d_q(in_data, out_data, out_grad, in_grad);
251  }
252 
253  std::vector<index3d<serial_size_t>> in_shape() const override {
254  if (params_.has_bias) {
255  return { params_.in, params_.weight,
256  index3d<serial_size_t>(1, 1, params_.out.depth_) };
257  } else {
258  return { params_.in, params_.weight };
259  }
260  }
261 
262  std::vector<index3d<serial_size_t>>
263  out_shape() const override { return { params_.out, params_.out }; }
264 
265  std::string layer_type() const override { return "q_conv"; }
266 
267  image<> weight_to_image() const {
268  image<> img;
269  const serial_size_t border_width = 1;
270  const auto pitch = params_.weight.width_ + border_width;
271  const auto width = params_.out.depth_ * pitch + border_width;
272  const auto height = params_.in.depth_ * pitch + border_width;
273  const image<>::intensity_t bg_color = 255;
274  const vec_t& W = *this->get_weights()[0];
275 
276  img.resize(width, height);
277  img.fill(bg_color);
278 
279  auto minmax = std::minmax_element(W.begin(), W.end());
280 
281  for (serial_size_t r = 0; r < params_.in.depth_; ++r) {
282  for (serial_size_t c = 0; c < params_.out.depth_; ++c) {
283  if (!params_.tbl.is_connected(c, r)) continue;
284 
285  const auto top = r * pitch + border_width;
286  const auto left = c * pitch + border_width;
287 
288  serial_size_t idx = 0;
289 
290  for (serial_size_t y = 0; y < params_.weight.height_; ++y) {
291  for (serial_size_t x = 0; x < params_.weight.width_; ++x) {
292  idx = c * params_.in.depth_ + r;
293  idx = params_.weight.get_index(x, y, idx);
294  const float_t w = W[idx];
295 
296  img.at(left + x, top + y)
297  = static_cast<image<>::intensity_t>(
298  rescale(w, *minmax.first,
299  *minmax.second, 0, 255));
300  }
301  }
302  }
303  }
304  return img;
305  }
306 
307  private:
308  void conv_set_params(const shape3d& in,
309  serial_size_t w_width,
310  serial_size_t w_height,
311  serial_size_t outc,
312  padding ptype,
313  bool has_bias,
314  serial_size_t w_stride,
315  serial_size_t h_stride,
316  const connection_table& tbl = connection_table()) {
317  params_.in = in;
318  params_.in_padded = shape3d(in_length(in.width_, w_width, ptype),
319  in_length(in.height_, w_height, ptype),
320  in.depth_);
321  params_.out =
322  shape3d(conv_out_length(in.width_, w_width, w_stride, ptype),
323  conv_out_length(in.height_, w_height, h_stride, ptype),
324  outc);
325  params_.weight = shape3d(w_width, w_height, in.depth_ * outc);
326  params_.has_bias = has_bias;
327  params_.pad_type = ptype;
328  params_.w_stride = w_stride;
329  params_.h_stride = h_stride;
330  params_.tbl = tbl;
331  }
332 
333  void init() {
334  if (params_.pad_type == padding::same) {
335  cws_.prev_out_buf_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
336  cws_.prev_delta_padded_.resize(1, vec_t(params_.in_padded.size(), float_t(0)));
337  }
338  else {
339  cws_.prev_out_buf_.clear();
340  }
341  }
342 
343  serial_size_t in_length(serial_size_t in_length,
344  serial_size_t window_size, padding pad_type) const {
345  return pad_type == padding::same ?
346  (in_length + window_size - 1) : in_length;
347  }
348 
349  static serial_size_t conv_out_length(serial_size_t in_length,
350  serial_size_t window_size,
351  serial_size_t stride, padding pad_type) {
352  float_t tmp;
353  if (pad_type == padding::same) {
354  tmp = static_cast<float_t>(in_length) / stride;
355  } else if (pad_type == padding::valid) {
356  tmp = static_cast<float_t>(in_length - window_size + 1) / stride;
357  } else {
358  throw nn_error("Not recognized pad_type.");
359  }
360  return static_cast<serial_size_t>(ceil(tmp));
361  }
362 
363  static serial_size_t conv_out_dim(serial_size_t in_width,
364  serial_size_t in_height,
365  serial_size_t window_size,
366  serial_size_t w_stride,
367  serial_size_t h_stride, padding pad_type) {
368  return conv_out_length(in_width, window_size, w_stride, pad_type) *
369  conv_out_length(in_height, window_size, h_stride, pad_type);
370  }
371 
372  serial_size_t conv_out_dim(serial_size_t in_width,
373  serial_size_t in_height,
374  serial_size_t window_width,
375  serial_size_t window_height,
376  serial_size_t w_stride,
377  serial_size_t h_stride, padding pad_type) const {
378  return conv_out_length(in_width, window_width, w_stride, pad_type) *
379  conv_out_length(in_height, window_height, h_stride, pad_type);
380  }
381 
382  void copy_and_pad_input(const tensor_t& in) {
384 
385  serial_size_t sample_count = static_cast<serial_size_t>(in.size());
386 
387  cws.prev_out_padded_.resize(sample_count);
388 
389  if (params_.pad_type == padding::same) {
390  cws.prev_out_buf_.resize(sample_count, cws.prev_out_buf_[0]);
391  cws.prev_delta_padded_.resize(sample_count, cws.prev_delta_padded_[0]);
392  }
393 
394  for (serial_size_t sample = 0; sample < sample_count; ++sample) {
395  if (params_.pad_type == padding::valid) {
396  cws.prev_out_padded_[sample] = &(in[sample]);
397  }
398  else {
399  vec_t* dst = &cws.prev_out_buf_[sample];
400 
401  // make padded version in order to avoid corner-case in fprop/bprop
402  for (serial_size_t c = 0; c < params_.in.depth_; c++) {
403  float_t *pimg = &(*dst)[params_.in_padded.get_index(params_.weight.width_ / 2, params_.weight.height_ / 2, c)];
404  const float_t *pin = &in[sample][params_.in.get_index(0, 0, c)];
405 
406  for (serial_size_t y = 0; y < params_.in.height_; y++, pin += params_.in.width_, pimg += params_.in_padded.width_) {
407  std::copy(pin, pin + params_.in.width_, pimg);
408  }
409  }
410 
411  cws.prev_out_padded_[sample] = &(cws.prev_out_buf_[sample]);
412  }
413  }
414  }
415 
416  void copy_and_unpad_delta(const tensor_t& delta, tensor_t& delta_unpadded) {
417  if (params_.pad_type == padding::valid) {
418  delta_unpadded = delta;
419  }
420  else {
421  for (serial_size_t sample = 0; sample < delta.size(); sample++) {
422  serial_size_t idx = 0;
423  const vec_t& src = delta[sample];
424  vec_t& dst = delta_unpadded[sample];
425 
426  for (serial_size_t c = 0; c < params_.in.depth_; c++) {
427  float_t *pdst = &dst[params_.in.get_index(0, 0, c)];
428  idx = params_.in_padded.get_index(params_.weight.width_ / 2,
429  params_.weight.height_ / 2, c);
430  const float_t *pin = &src[idx];
431 
432  for (serial_size_t y = 0; y < params_.in.height_; y++) {
433  std::copy(pin, pin + params_.in.width_, pdst);
434  pdst += params_.in.width_;
435  pin += params_.in_padded.width_;
436  }
437  }
438  }
439  }
440  }
441 
442  void init_backend(const backend_t backend_type) {
443  std::shared_ptr<core::backend> backend = nullptr;
444 
445  // allocate new backend
446  if (backend_type == backend_t::internal) {
447  backend = std::make_shared<core::tiny_backend>(&params_,
448  [this](const tensor_t& in) {
449  return copy_and_pad_input(in);
450  },
451  [this](const tensor_t& delta, tensor_t& dst) {
452  return copy_and_unpad_delta(delta, dst);
453  },
454  [this](const tensor_t& p_delta,
455  const tensor_t& out, tensor_t& c_delta) {
456  return Base::backward_activation(p_delta, out, c_delta);
457  },
458  &cws_);
459  } else if (backend_type == backend_t::nnpack) {
460  backend = std::make_shared<core::nnp_backend>(&params_,
461  [this](const tensor_t& in) {
462  return copy_and_pad_input(in);
463  },
464  &cws_);
465  } else if (backend_type == backend_t::libdnn) {
466  backend = std::make_shared<core::dnn_backend>();
467 #ifdef CNN_USE_AVX
468  } else if (backend_type == backend_t::avx) {
469  backend = std::make_shared<core::avx_backend>(&params_,
470  [this](const tensor_t& in) {
471  return copy_and_pad_input(in);
472  },
473  [this](const tensor_t& delta, tensor_t& dst) {
474  return copy_and_unpad_delta(delta, dst);
475  },
476  [this](const tensor_t& p_delta,
477  const tensor_t& out, tensor_t& c_delta) {
478  return Base::backward_activation(p_delta, out, c_delta);
479  },
480  &cws_);
481 #endif
482  } else {
483  throw nn_error("Not supported backend type.");
484  }
485 
486  if (backend) {
487  Base::set_backend(backend);
488  Base::backend_->set_layer(this);
489  } else {
490  throw nn_error("Could not allocate the backend.");
491  }
492  }
493 
494  /* The convolution parameters */
495  conv_params params_;
496 
497  /* The type of backend */
498  //backend_t backend_type_;
499 
500  /* Workers buffers */
502 };
503 
504 } // namespace tiny_dnn
Definition: backend.h:68
Definition: conv_params.h:92
single-input, single-output network with activation function
Definition: feedforward_layer.h:37
Simple image utility class.
Definition: image.h:94
2D convolution layer
Definition: quantized_convolutional_layer.h:54
quantized_convolutional_layer(quantized_convolutional_layer &&other)
number of incoming connections for each output unit
Definition: quantized_convolutional_layer.h:202
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition: quantized_convolutional_layer.h:143
void back_propagation(const std::vector< tensor_t * > &in_data, const std::vector< tensor_t * > &out_data, std::vector< tensor_t * > &out_grad, std::vector< tensor_t * > &in_grad) override
return delta of previous layer (delta=\frac{dE}{da}, a=wx in fully-connected layer)
Definition: quantized_convolutional_layer.h:246
std::vector< index3d< serial_size_t > > out_shape() const override
array of output shapes (width x height x depth)
Definition: quantized_convolutional_layer.h:263
serial_size_t fan_out_size() const override
number of outgoing connections for each input unit used only for weight/bias initialization methods w...
Definition: quantized_convolutional_layer.h:216
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_size, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition: quantized_convolutional_layer.h:74
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition: quantized_convolutional_layer.h:108
std::string layer_type() const override
name of layer, should be unique for each concrete class
Definition: quantized_convolutional_layer.h:265
void forward_propagation(const std::vector< tensor_t * > &in_data, std::vector< tensor_t * > &out_data) override
Definition: quantized_convolutional_layer.h:226
quantized_convolutional_layer(serial_size_t in_width, serial_size_t in_height, serial_size_t window_width, serial_size_t window_height, serial_size_t in_channels, serial_size_t out_channels, const connection_table &connection_table, padding pad_type=padding::valid, bool has_bias=true, serial_size_t w_stride=1, serial_size_t h_stride=1, backend_t backend_type=core::backend_t::internal)
constructing convolutional layer
Definition: quantized_convolutional_layer.h:180
serial_size_t fan_in_size() const override
number of outgoing connections for each input unit
Definition: quantized_convolutional_layer.h:210
std::vector< index3d< serial_size_t > > in_shape() const override
array of input shapes (width x height x depth)
Definition: quantized_convolutional_layer.h:253
Definition: conv_params.h:40