tiny_dnn  1.0.0
A header only, dependency-free deep learning framework in C++11
activation_function.h
1 /*
2  Copyright (c) 2013, Taiga Nomi
3  All rights reserved.
4 
5  Redistribution and use in source and binary forms, with or without
6  modification, are permitted provided that the following conditions are met:
7  * Redistributions of source code must retain the above copyright
8  notice, this list of conditions and the following disclaimer.
9  * Redistributions in binary form must reproduce the above copyright
10  notice, this list of conditions and the following disclaimer in the
11  documentation and/or other materials provided with the distribution.
12  * Neither the name of the <organization> nor the
13  names of its contributors may be used to endorse or promote products
14  derived from this software without specific prior written permission.
15 
16  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
17  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 #pragma once
28 #include "tiny_dnn/util/util.h"
29 #include <algorithm>
30 
31 namespace tiny_dnn {
32 namespace activation {
33 
34 class function {
35 public:
36  function() = default;
37  function(const function &) = default;
38 #ifndef CNN_DEFAULT_MOVE_CONSTRUCTOR_UNAVAILABLE
39  function(function &&) = default;
40 #endif
41  function &operator =(const function &) = default;
42 #ifndef CNN_DEFAULT_ASSIGNMENT_OPERATOR_UNAVAILABLE
43  function &operator =(function &&) = default;
44 #endif
45  virtual ~function() = default;
46 
47  virtual float_t f(const vec_t& v, size_t index) const = 0;
48  void itef(vec_t& out, const vec_t& in, size_t cnt) const {
49  for (size_t i = 0; i < cnt; i++) {
50  out[i] = f(in, i);
51  }
52  }
53 
54  // dfi/dyi
55  virtual float_t df(float_t y) const = 0;
56 
57  // dfi/dyk (k=0,1,..n)
58  virtual vec_t df(const vec_t& y, size_t i) const { vec_t v(y.size(), 0); v[i] = df(y[i]); return v; }
59 
60  // return if dfi/dyk is one-hot vector
61  virtual bool one_hot() const { return true; }
62 
63  // target value range for learning
64  virtual std::pair<float_t, float_t> scale() const = 0;
65 };
66 
67 class identity : public function {
68 public:
69  using function::df;
70  float_t f(const vec_t& v, size_t i) const override { return v[i]; }
71  float_t df(float_t /*y*/) const override { return float_t(1); }
72  std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
73 };
74 
75 class sigmoid : public function {
76 public:
77  using function::df;
78  float_t f(const vec_t& v, size_t i) const override { return float_t(1) / (float_t(1) + std::exp(-v[i])); }
79  float_t df(float_t y) const override { return y * (float_t(1) - y); }
80  std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
81 };
82 
83 class relu : public function {
84 public:
85  using function::df;
86  float_t f(const vec_t& v, size_t i) const override { return std::max(float_t(0), v[i]); }
87  float_t df(float_t y) const override { return y > float_t(0) ? float_t(1) : float_t(0); }
88  std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
89 };
90 
91 typedef relu rectified_linear; // for compatibility
92 
93 class leaky_relu : public function {
94 public:
95  using function::df;
96  float_t f(const vec_t& v, size_t i) const override { return (v[i] > float_t(0)) ? v[i] : float_t(0.01) * v[i]; }
97  float_t df(float_t y) const override { return y > float_t(0) ? float_t(1) : float_t(0.01); }
98  std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
99 };
100 
101 class elu : public function {
102 public:
103  using function::df;
104  float_t f(const vec_t& v, size_t i) const override { return (v[i]<float_t(0) ? (exp(v[i])- float_t(1)) : v[i]); }
105  float_t df(float_t y) const override { return (y > float_t(0) ? float_t(1) : (float_t(1)+y)); }
106  std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
107 };
108 
109 class softmax : public function {
110 public:
111  float_t f(const vec_t& v, size_t i) const override {
112  float_t alpha = *std::max_element(v.begin(), v.end());
113  float_t numer = std::exp(v[i] - alpha);
114  float_t denom = float_t(0);
115  for (auto x : v)
116  denom += std::exp(x - alpha);
117  return numer / denom;
118  }
119 
120  float_t df(float_t y) const override {
121  return y * (float_t(1) - y);
122  }
123 
124  virtual vec_t df(const vec_t& y, size_t index) const override {
125  vec_t v(y.size(), 0);
126  for (size_t i = 0; i < y.size(); i++)
127  v[i] = (i == index) ? df(y[index]) : -y[i] * y[index];
128 
129  return v;
130  }
131 
132  virtual bool one_hot() const override { return false; }
133 
134  std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0), float_t(1)); }
135 };
136 
137 class tan_h : public function {
138 public:
139  using function::df;
140 
141  float_t f(const vec_t& v, size_t i) const override {
142  return std::tanh(v[i]);
143  }
144 
145  void itef(vec_t& out, const vec_t& in, size_t cnt) const {
146  for (size_t i = 0; i < cnt; i++) {
147  out[i] = std::tanh(in[i]);
148  }
149  }
150 
151  // fast approximation of tanh (improve 2-3% speed in LeNet-5)
152  /*float_t f(float_t x) const {
153  const float_t x2 = x * x;
154  x *= 1.0 + x2 * (0.1653 + x2 * 0.0097);
155  return x / std::sqrt(1.0 + x * x);// invsqrt(static_cast<float>(1.0 + x * x));
156  }*/
157 
158  float_t df(float_t y) const override { return float_t(1) - sqr(y); }
159  std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(-0.8), float_t(0.8)); }
160 
161 private:
162  /*float invsqrt(float x) const {
163  float x2 = x * 0.5f;
164  long i = *reinterpret_cast<long*>(&x);
165 
166  i = 0x5f3759df - (i >> 1);
167  x = *reinterpret_cast<float*>(&i);
168  x = x * (1.5f - (x2 * x * x));
169  return x;
170  }*/
171 };
172 
173 // s tan_h, but scaled to match the other functions
174 class tan_hp1m2 : public function {
175 public:
176  using function::df;
177  float_t f(const vec_t& v, size_t i) const override {
178  const float_t ep = std::exp(v[i]);
179  return ep / (ep + std::exp(-v[i]));
180  }
181 
182  float_t df(float_t y) const override { return 2 * y *(float_t(1) - y); }
183  std::pair<float_t, float_t> scale() const override { return std::make_pair(float_t(0.1), float_t(0.9)); }
184 };
185 
186 } // namespace activation
187 } // namespace tiny_dnn
Definition: activation_function.h:101
Definition: activation_function.h:34
Definition: activation_function.h:67
Definition: activation_function.h:93
Definition: activation_function.h:83
Definition: activation_function.h:75
Definition: activation_function.h:109
Definition: activation_function.h:137
Definition: activation_function.h:174