深度学习中的优化算法之Adam

转x33g5p2x  于2022-05-28 转载在 其他  
字(5.4k)|赞(0)|评价(0)|浏览(327)

    之前在https://blog.csdn.net/fengbingchun/article/details/124909910 介绍过深度学习中的优化算法Adadelta，这里介绍下深度学习的另一种优化算法Adam。论文名字为《ADAM: A METHOD FOR STOCHASTIC OPTIMIZATION》，论文地址：https://arxiv.org/pdf/1412.6980.pdf

    Adam(Adaptive Moment Estimation)：自适应矩估计是一种计算每个参数或权重的自适应学习率的方法，它除了像Adadelta和RMSProp一样存储exponentially decaying average of past squared gradients外，Adam还保持exponentially decaying average of past gradients，类似于动量(momentum)。如下图所示，截图来自：https://arxiv.org/pdf/1609.04747.pdf

    Adam是梯度下降优化算法的扩展，当处理涉及大量数据或大量参数时，该算法非常有效。直观地说，它是带动量的梯度下降算法和RMSProp算法的结合。

    Adam旨在加速优化过程，例如减少达到最优值所需的迭代次数，或提高优化算法的能力，例如获得更好的最终结果。

    以下是与Adadelta不同的代码片段：

    1.在原有枚举类Optimizaiton的基础上新增Adam：

  1. enum class Optimization {
  2. 	BGD, // Batch Gradient Descent
  3. 	SGD, // Stochastic Gradient Descent
  4. 	MBGD, // Mini-batch Gradient Descent
  5. 	SGD_Momentum, // SGD with Momentum
  6. 	AdaGrad, // Adaptive Gradient
  7. 	RMSProp, // Root Mean Square Propagation
  8. 	Adadelta, // an adaptive learning rate method
  9. 	Adam // Adaptive Moment Estimation
  10. };

    2.calculate_gradient_descent函数：

  1. void LogisticRegression2::calculate_gradient_descent(int start, int end)
  2. {
  3. 	switch (optim_) {
  4. 		case Optimization::Adam: {
  5. 			int len = end - start;
  6. 			std::vector<float> m(feature_length_, 0.), v(feature_length_, 0.), mhat(feature_length_, 0.), vhat(feature_length_, 0.);
  7. 			std::vector<float> z(len, 0.), dz(len, 0.);
  8. 			float beta1t = 1., beta2t = 1.;
  9. 			for (int i = start, x = 0; i < end; ++i, ++x) {
  10. 				z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
  11. 				dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);
  13. 				beta1t *= beta1_;
  14. 				beta2t *= beta2_;
  16. 				for (int j = 0; j < feature_length_; ++j) {
  17. 					float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
  18. 					m[j] = beta1_ * m[j] + (1. - beta1_) * dw; // formula 19
  19. 					v[j] = beta2_ * v[j] + (1. - beta2_) * (dw * dw); // formula 19
  21. 					mhat[j] = m[j] / (1. - beta1t); // formula 20
  22. 					vhat[j] = v[j] / (1. - beta2t); // formula 20
  24. 					w_[j] = w_[j] - alpha_ * mhat[j] / (std::sqrt(vhat[j]) + eps_); // formula 21
  25. 				}
  27. 				b_ -= (alpha_ * dz[x]);
  28. 			}
  29. 		}
  30. 			break;
  31. 		case Optimization::Adadelta: {
  32. 			int len = end - start;
  33. 			std::vector<float> g(feature_length_, 0.), p(feature_length_, 0.);
  34. 			std::vector<float> z(len, 0.), dz(len, 0.);
  35. 			for (int i = start, x = 0; i < end; ++i, ++x) {
  36. 				z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
  37. 				dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);
  39. 				for (int j = 0; j < feature_length_; ++j) {
  40. 					float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
  41. 					g[j] = mu_ * g[j] + (1. - mu_) * (dw * dw); // formula 10
  43. 					float alpha = (eps_ + std::sqrt(p[j])) / (eps_ + std::sqrt(g[j]));
  44. 					float change = alpha * dw;
  45. 					p[j] = mu_ * p[j] +  (1. - mu_) * (change * change); // formula 15
  47. 					w_[j] = w_[j] - change;
  48. 				}
  50. 				b_ -= (eps_ * dz[x]);
  51. 			}
  52. 		}
  53. 			break;
  54. 		case Optimization::RMSProp: {
  55. 			int len = end - start;
  56. 			std::vector<float> g(feature_length_, 0.);
  57. 			std::vector<float> z(len, 0), dz(len, 0);
  58. 			for (int i = start, x = 0; i < end; ++i, ++x) {
  59. 				z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
  60. 				dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);
  62. 				for (int j = 0; j < feature_length_; ++j) {
  63. 					float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
  64. 					g[j] = mu_ * g[j] + (1. - mu_) * (dw * dw); // formula 18
  65. 					w_[j] = w_[j] - alpha_ * dw / (std::sqrt(g[j]) + eps_);
  66. 				}
  68. 				b_ -= (alpha_ * dz[x]);
  69. 			}
  70. 		}
  71. 			break;
  72. 		case Optimization::AdaGrad: {
  73. 			int len = end - start;
  74. 			std::vector<float> g(feature_length_, 0.);
  75. 			std::vector<float> z(len, 0), dz(len, 0);
  76. 			for (int i = start, x = 0; i < end; ++i, ++x) {
  77. 				z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
  78. 				dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);
  80. 				for (int j = 0; j < feature_length_; ++j) {
  81. 					float dw = data_->samples[random_shuffle_[i]][j] * dz[x];
  82. 					g[j] += dw * dw;
  83. 					w_[j] = w_[j] - alpha_ * dw / (std::sqrt(g[j]) + eps_);
  84. 				}
  86. 				b_ -= (alpha_ * dz[x]);
  87. 			}
  88. 		}
  89. 			break;
  90. 		case Optimization::SGD_Momentum: {
  91. 			int len = end - start;
  92. 			std::vector<float> change(feature_length_, 0.);
  93. 			std::vector<float> z(len, 0), dz(len, 0);
  94. 			for (int i = start, x = 0; i < end; ++i, ++x) {
  95. 				z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
  96. 				dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);
  98. 				for (int j = 0; j < feature_length_; ++j) {
  99. 					float new_change = mu_ * change[j] - alpha_ * (data_->samples[random_shuffle_[i]][j] * dz[x]);
  100. 					w_[j] += new_change;
  101. 					change[j] = new_change;
  102. 				}
  104. 				b_ -= (alpha_ * dz[x]);
  105. 			}
  106. 		}
  107. 			break;
  108. 		case Optimization::SGD:
  109. 		case Optimization::MBGD: {
  110. 			int len = end - start;
  111. 			std::vector<float> z(len, 0), dz(len, 0);
  112. 			for (int i = start, x = 0; i < end; ++i, ++x) {
  113. 				z[x] = calculate_z(data_->samples[random_shuffle_[i]]);
  114. 				dz[x] = calculate_loss_function_derivative(calculate_activation_function(z[x]), data_->labels[random_shuffle_[i]]);
  116. 				for (int j = 0; j < feature_length_; ++j) {
  117. 					w_[j] = w_[j] - alpha_ * (data_->samples[random_shuffle_[i]][j] * dz[x]);
  118. 				}
  120. 				b_ -= (alpha_ * dz[x]);
  121. 			}
  122. 		}
  123. 			break;
  124. 		case Optimization::BGD:
  125. 		default: // BGD
  126. 			std::vector<float> z(m_, 0), dz(m_, 0);
  127. 			float db = 0.;
  128. 			std::vector<float> dw(feature_length_, 0.);
  129. 			for (int i = 0; i < m_; ++i) {
  130. 				z[i] = calculate_z(data_->samples[i]);
  131. 				o_[i] = calculate_activation_function(z[i]);
  132. 				dz[i] = calculate_loss_function_derivative(o_[i], data_->labels[i]);
  134. 				for (int j = 0; j < feature_length_; ++j) {
  135. 					dw[j] += data_->samples[i][j] * dz[i]; // dw(i)+=x(i)(j)*dz(i)
  136. 				}
  137. 				db += dz[i]; // db+=dz(i)
  138. 			}
  140. 			for (int j = 0; j < feature_length_; ++j) {
  141. 				dw[j] /= m_;
  142. 				w_[j] -= alpha_ * dw[j];
  143. 			}
  145. 			b_ -= alpha_*(db/m_);
  146. 	}
  147. }

    执行结果如下图所示：测试函数为test_logistic_regression2_gradient_descent，多次执行每种配置，最终结果都相同。图像集使用MNIST，其中训练图像总共10000张，0和1各5000张，均来自于训练集；预测图像总共1800张，0和1各900张，均来自于测试集。在Adadelta中设置eps为1e-3，耗时为26秒；在Adam中设置eps为1e-8，学习率为0.001，beta1为0.8，beta2为0.888时，耗时为27秒；它们的识别率均为100%

    GitHub： https://github.com/fengbingchun/NN_Test

版权说明 : 本文为转载文章, 版权归原作者所有 版权申明

原文链接 : https://blog.csdn.net/fengbingchun/article/details/125018001

内容来源于网络，如有侵权，请联系作者删除！

Adam

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com