深度神经网络中的Batch Normalization介绍及实现

转x33g5p2x  于2021-11-11 转载在 其他  
字(7.5k)|赞(0)|评价(0)|浏览(337)

    之前在https://blog.csdn.net/fengbingchun/article/details/114493591中介绍DenseNet时，网络中会有BN层，即Batch Normalization，在每个Dense Block中都会有BN参与运算，下面对BN进行介绍并给出C++和PyTorch实现。

    Batch Normalization即批量归一化由Sergey loffe等人于2015年提出，论文名为：《Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift》，论文见：https://arxiv.org/pdf/1502.03167.pdf 。

    Batch Normalization是一种算法方法，它使深度神经网络的训练更快、更稳定。它可在激活函数前也可在激活函数后进行。它依赖于batch size，当batch size较小时，性能退化严重。在训练和测试阶段，它的计算方式不同。

    对于CNN，使用BN更好；对于RNN，使用LN(Layer Normalization)更好。

    在训练过程中，由于每层输入的分布随着前一层的参数发生变化而发生变化，因此训练深度神经网络很复杂。由于需要较低的学习率和仔细的参数初始化，这会减慢训练速度，并且使得训练具有饱和非线性的模型变得非常困难。我们将这种现象称为内部协变量偏移(internal covariate shift)，并通过归一化层输入来解决该问题。

    Batch Normalization用于训练小批量样本(mini-batch)。它允许我们使用更高的学习率，并且不必太小心初始化。它还充当正则化器，在某些情况下消除了Dropout的需要。

    Batch Normalization实现算法如下，截图来自原始论文：

    在一个mini-batch中，在每一个BN层中，对每个样本的同一通道，计算它们的均值和方差，再对数据进行归一化，归一化到平均值为0，标准差为1 的常态分布，最后使用两个可学习参数gamma和beta对归一化的数据进行缩放和移位。此外，在训练过程中还保存了每个mini-batch每一BN层的均值和方差，最后求所有mini-batch均值和方差的期望值，以此来作为推理过程中该BN层的均值和方差。

    Batch Normalization优点：

    (1).在不影响收敛性的情况下，可使用更大的学习率，使训练更快、更稳定；

    (2).具有正则化效果，防止过拟合，可去除Dropout和局部响应归一化(Local Response Normalization, LRN)；

    (3).由于训练数据打乱顺序，使得每个epoch中mini-batch都不一样，对不同mini-batch做归一化可以起到数据增强的效果；

    (4).缓减梯度爆炸和梯度消失。

    以下是C++实现：

    batch_normalization.hpp:

  1. #ifndef FBC_SRC_NN_BATCH_NORM_HPP_
  2. #define FBC_SRC_NN_BATCH_NORM_HPP_
  4. #include <vector>
  5. #include <memory>
  7. // Blog: 
  9. namespace ANN {
  11. class BatchNorm {
  12. public:
  13. 	BatchNorm(int number, int channels, int height, int width) : number_(number), channels_(channels), height_(height), width_(width) {}
  14. 	int LoadData(const float* data, int length);
  15. 	std::unique_ptr<float []> Run();
  17. 	void SetGamma(float gamma) { gamma_ = gamma; }
  18. 	float GetGamma() const { return gamma_; }
  19. 	void SetBeta(float beta) { beta_ = beta; }
  20. 	float GetBeta() const { return beta_; }
  21. 	void SetMean(std::vector<float> mean) { mean_ = mean; }
  22. 	std::vector<float> GetMean() const { return mean_; }
  23. 	void SetVariance(std::vector<float> variance) { variance_ = variance; }
  24. 	std::vector<float> GetVariance() const { return variance_; }
  25. 	void SetEpsilon(float epsilon) { epsilon_ = epsilon; }
  27. private:
  28. 	int number_; // mini-batch
  29. 	int channels_;
  30. 	int height_;
  31. 	int width_;
  32. 	std::vector<float> mean_;
  33. 	std::vector<float> variance_;
  34. 	float gamma_  = 1.;
  35. 	float beta_ = 0.;
  36. 	float epsilon_ = 1e-5;
  37. 	std::vector<float> data_;
  38. };
  40. } // namespace ANN
  42. #endif // FBC_SRC_NN_BATCH_NORM_HPP_

    batch_normalization.cpp:

  1. #include "batch_normalization.hpp"
  2. #include <string.h>
  3. #include <vector>
  4. #include <cmath>
  5. #include "common.hpp"
  7. namespace ANN {
  9. int BatchNorm::LoadData(const float* data, int length)
  10. {
  11. 	CHECK(number_ * channels_ * height_ * width_ == length);
  13. 	data_.resize(length);
  14. 	memcpy(data_.data(), data, length * sizeof(float));
  15. 	return 0;
  16. }
  18. std::unique_ptr<float[]> BatchNorm::Run()
  19. {
  20. 	mean_.resize(channels_ * height_ * width_);
  21. 	memset(mean_.data(), 0, mean_.size() * sizeof(float));
  23. 	for (int n = 0; n < number_; ++n) {
  24. 		const float* p = data_.data() + n * (channels_ * height_ * width_);
  25. 		for (int c = 0; c < channels_; ++c) {
  26. 			for (int h = 0; h < height_; ++h) {
  27. 				for (int w = 0; w < width_; ++w) {
  28. 					mean_[c * height_ * width_ + h * width_ + w] += p[c * height_ * width_ + h * width_ + w];
  29. 				}
  30. 			}
  31. 		}
  32. 	}
  34. 	for (int len = 0; len < channels_ * height_ * width_; ++len) {
  35. 		mean_[len] /= number_;
  36. 	}
  38. 	variance_.resize(channels_ * height_ * width_);
  39. 	memset(variance_.data(), 0, variance_.size() * sizeof(float));
  41. 	for (int n = 0; n < number_; ++n) {
  42. 		const float* p = data_.data() + n * (channels_ * height_ * width_);
  43. 		for (int c = 0; c < channels_; ++c) {
  44. 			for (int h = 0; h < height_; ++h) {
  45. 				for (int w = 0; w < width_; ++w) {
  46. 					variance_[c * height_ * width_ + h * width_ + w] += std::pow(p[c * height_ * width_ + h * width_ + w] - mean_[c * height_ * width_ + h * width_ + w], 2.);
  47. 				}
  48. 			}
  49. 		}
  50. 	}
  52. 	for (int len = 0; len < channels_ * height_ * width_; ++len) {
  53. 		variance_[len] /= number_;
  54. 	}
  56. 	std::unique_ptr<float[]> output(new float[number_ * channels_ * height_ * width_]);
  57. 	for (int n = 0; n < number_; ++n) {
  58. 		const float* p1 = data_.data() + n * (channels_ * height_ * width_);
  59. 		float* p2 = output.get() + n * (channels_ * height_ * width_);
  61. 		for (int c = 0; c < channels_; ++c) {
  62. 			for (int h = 0; h < height_; ++h) {
  63. 				for (int w = 0; w < width_; ++w) {
  64. 					p2[c * height_ * width_ + h * width_ + w] = (p1[c * height_ * width_ + h * width_ + w] - mean_[c * height_ * width_ + h * width_ + w]) /
  65. 						std::sqrt(variance_[c * height_ * width_ + h * width_ + w] + epsilon_);
  66. 				}
  67. 			}
  68. 		}
  69. 	}
  71. 	return output;
  72. }
  74. } // namespace ANN

    funset.cpp:

  1. int test_batch_normalization()
  2. {
  3. 	const std::vector<float> data = { 11.1, -2.2, 23.3, 54.4, 58.5, -16.6,
  4. 									-97.7, -28.8, 49.9, -61.3, 52.6, -33.9,
  5. 									-2.45, -15.7, 72.4, 9.1, 47.2, 21.7};
  6. 	const int number = 3, channels = 1, height = 1, width = 6;
  8. 	ANN::BatchNorm bn(number, channels, height, width);
  9. 	bn.LoadData(data.data(), data.size());
  11. 	std::unique_ptr<float[]> output = bn.Run();
  12. 	
  13. 	fprintf(stdout, "result:\n");
  14. 	for (int n = 0; n < number; ++n) {
  15. 		const float* p = output.get() + n * (channels * height * width);
  16. 		for (int c = 0; c < channels; ++c) {
  17. 			for (int h = 0; h < height; ++h) {
  18. 				for (int w = 0; w < width; ++w) {
  19. 					fprintf(stdout, "%f, ", p[c * (height * width) + h * width + w]);
  20. 				}
  21. 				fprintf(stdout, "\n");
  22. 			}
  23. 		}
  24. 	}
  26. 	return 0;
  27. }

    执行结果如下：

    以下是调用PyTorch接口实现：test_batch_normalization.py

  1. import torch
  2. from torch import nn
  3. import numpy as np
  5. # reference: https://github.com/Johann-Huber/batchnorm_pytorch/blob/main/batch_normalization_in_pytorch.ipynb
  7. # BatchNorm reimplementation
  8. class myBatchNorm2d(nn.Module):
  9.     def __init__(self, input_size = None , epsilon = 1e-5, momentum = 0.99):
  10.         super(myBatchNorm2d, self).__init__()
  11.         assert input_size, print('Missing input_size parameter.')
  13.         # Batch mean & var must be defined during training
  14.         self.mu = torch.zeros(1, input_size)
  15.         self.var = torch.ones(1, input_size)
  17.         # For numerical stability
  18.         self.epsilon = epsilon
  20.         # Exponential moving average for mu & var update
  21.         self.it_call = 0  # training iterations
  22.         self.momentum = momentum # EMA smoothing
  24.         # Trainable parameters
  25.         self.beta = torch.nn.Parameter(torch.zeros(1, input_size))
  26.         self.gamma = torch.nn.Parameter(torch.ones(1, input_size))
  28.         # Batch size on which the normalization is computed
  29.         self.batch_size = 0
  31.     def forward(self, x):
  32.         # [batch_size, input_size]
  34.         self.it_call += 1
  36.         if self.training:
  37.             print("Info: training ...")
  38.             if( self.batch_size == 0 ):
  39.                 # First iteration : save batch_size
  40.                 self.batch_size = x.shape[0]
  42.             # Training : compute BN pass
  43.             #batch_mu = (x.sum(dim=0)/x.shape[0]).unsqueeze(0) # [1, input_size]
  44.             batch_mu = torch.mean(x, dim=0)
  45.             #batch_var = (x.var(dim=0)/x.shape[0]).unsqueeze(0)*2 # [1, input_size]
  46.             batch_var = torch.var(x, unbiased=False, dim=0)
  47.             #print("batch_mu:", batch_mu)
  48.             #print("batch_var:", batch_var)
  50.             x_normalized = (x-batch_mu)/torch.sqrt(batch_var + self.epsilon) # [batch_size, input_size]
  51.             x_bn = self.gamma * x_normalized + self.beta # [batch_size, input_size]
  53.             # Update mu & std
  54.             if(x.shape[0] == self.batch_size):
  55.                 running_mu = batch_mu
  56.                 running_var = batch_var
  57.             else:
  58.                 running_mu = batch_mu*self.batch_size/x.shape[0]
  59.                 running_var = batch_var*self.batch_size/x.shape[0]
  61.             self.mu = running_mu * (self.momentum/self.it_call) + \
  62.                             self.mu * (1 - (self.momentum/self.it_call))
  63.             self.var = running_var * (self.momentum/self.it_call) + \
  64.                         self.var * (1 - (self.momentum/self.it_call))
  66.         else:
  67.             print("Info: inference ...")
  68.             # Inference: compute BN pass using estimated mu & var
  69.             if (x.shape[0] == self.batch_size):
  70.                 estimated_mu = self.mu
  71.                 estimated_var = self.var
  72.             else :
  73.                 estimated_mu = self.mu*x.shape[0]/self.batch_size
  74.                 estimated_var = self.var*x.shape[0]/self.batch_size
  76.             x_normalized = (x-estimated_mu)/torch.sqrt(estimated_var + self.epsilon) # [batch_size, input_size]
  77.             x_bn = self.gamma * x_normalized + self.beta # [batch_size, input_size]
  79.         return x_bn # [batch_size, output_size=input_size]
  81. # N = 3, C = 1, H = 1, W = 6
  82. input_size = 1 # channel
  83. bn = myBatchNorm2d(input_size)
  84. data =  [[[[11.1, -2.2, 23.3, 54.4, 58.5, -16.6]]],
  85. 		[[[-97.7, -28.8, 49.9, -61.3, 52.6, -33.9]]],
  86. 		[[[-2.45, -15.7, 72.4, 9.1, 47.2, 21.7]]]]
  87. input = torch.FloatTensor(data) # [N, C, H, W]
  88. print("input:", input)
  89. output = bn.forward(input)
  90. print("output:", output)
  92. '''
  93. print("######################")
  94. a = np.array(data)
  95. print(np.mean(a, axis=0))
  96. print(np.var(a, axis=0))
  97. '''

    执行结果如下：可见，C++和PyTorch实现结果相同

    GitHub:

    https://github.com/fengbingchun/NN_Test

    https://github.com/fengbingchun/PyTorch_Test

版权说明 : 本文为转载文章, 版权归原作者所有 版权申明

原文链接 : https://blog.csdn.net/fengbingchun/article/details/118959997

内容来源于网络，如有侵权，请联系作者删除！

pytorchbn

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com