机器学习进阶笔记之五 | 深入理解VGG\Residual Network

Deep Residual Network tflearn实现

      tflearn官方有一个cifar10的实现， 代码如下：

1.  from __future__ import division, print_function, absolute_import
2. 
   
3.  import tflearn
4. 
   
5.  # Residual blocks
6.  # 32 layers: n=5, 56 layers: n=9, 110 layers: n=18
7.  n = 5
8. 
   
9.  # Data loading
10.  from tflearn.datasets import cifar10
11.  (X, Y), (testX, testY) = cifar10.load_data()
12.  Y = tflearn.data_utils.to_categorical(Y, 10)
13.  testY = tflearn.data_utils.to_categorical(testY, 10)
14. 
    
15.  # Real-time data preprocessing
16.  img_prep = tflearn.ImagePreprocessing()
17.  img_prep.add_featurewise_zero_center(per_channel=True)
18. 
    
19.  # Real-time data augmentation
20.  img_aug = tflearn.ImageAugmentation()
21.  img_aug.add_random_flip_leftright()
22.  img_aug.add_random_crop([32, 32], padding=4)
23. 
    
24.  # Building Residual Network
25.  net = tflearn.input_data(shape=[None, 32, 32, 3],
26.                           data_preprocessing=img_prep,
27.                           data_augmentation=img_aug)
28.  net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001)
29.  net = tflearn.residual_block(net, n, 16)
30.  net = tflearn.residual_block(net, 1, 32, downsample=True)
31.  net = tflearn.residual_block(net, n-1, 32)
32.  net = tflearn.residual_block(net, 1, 64, downsample=True)
33.  net = tflearn.residual_block(net, n-1, 64)
34.  net = tflearn.batch_normalization(net)
35.  net = tflearn.activation(net, 'relu')
36.  net = tflearn.global_avg_pool(net)
37.  # Regression
38.  net = tflearn.fully_connected(net, 10, activation='softmax')
39.  mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
40.  net = tflearn.regression(net, optimizer=mom,
41.                           loss='categorical_crossentropy')
42.  # Training
43.  model = tflearn.DNN(net, checkpoint_path='model_resnet_cifar10',
44.                      max_checkpoints=10, tensorboard_verbose=0,
45.                      clip_gradients=0.)
46. 
    
47.  model.fit(X, Y, n_epoch=200, validation_set=(testX, testY),
48.            snapshot_epoch=False, snapshot_step=500,
49.            show_metric=True, batch_size=128, shuffle=True,
50.            run_id='resnet_cifar10')

       其中，residual_block实现了shortcut，代码写的十分棒：

1.  def residual_block(incoming, nb_blocks, out_channels, downsample=False,
2.                     downsample_strides=2, activation='relu', batch_norm=True,
3.                     bias=True, weights_init='variance_scaling',
4.                     bias_init='zeros', regularizer='L2', weight_decay=0.0001,
5.                     trainable=True, restore=True, reuse=False, scope=None,
6.                     name="ResidualBlock"):
7.      """ Residual Block.
8. 
   
9.      A residual block as described in MSRA's Deep Residual Network paper.
10.      Full pre-activation architecture is used here.
11. 
    
12.      Input:
13.          4-D Tensor [batch, height, width, in_channels].
14. 
    
15.      Output:
16.          4-D Tensor [batch, new height, new width, nb_filter].
17. 
    
18.      Arguments:
19.          incoming: `Tensor`. Incoming 4-D Layer.
20.          nb_blocks: `int`. Number of layer blocks.
21.          out_channels: `int`. The number of convolutional filters of the
22.              convolution layers.
23.          downsample: `bool`. If True, apply downsampling using
24.              'downsample_strides' for strides.
25.          downsample_strides: `int`. The strides to use when downsampling.
26.          activation: `str` (name) or `function` (returning a `Tensor`).
27.              Activation applied to this layer (see tflearn.activations).
28.              Default: 'linear'.
29.          batch_norm: `bool`. If True, apply batch normalization.
30.          bias: `bool`. If True, a bias is used.
31.          weights_init: `str` (name) or `Tensor`. Weights initialization.
32.              (see tflearn.initializations) Default: 'uniform_scaling'.
33.          bias_init: `str` (name) or `tf.Tensor`. Bias initialization.
34.              (see tflearn.initializations) Default: 'zeros'.
35.          regularizer: `str` (name) or `Tensor`. Add a regularizer to this
36.              layer weights (see tflearn.regularizers). Default: None.
37.          weight_decay: `float`. Regularizer decay parameter. Default: 0.001.
38.          trainable: `bool`. If True, weights will be trainable.
39.          restore: `bool`. If True, this layer weights will be restored when
40.              loading a model.
41.          reuse: `bool`. If True and 'scope' is provided, this layer variables
42.              will be reused (shared).
43.          scope: `str`. Define this layer scope (optional). A scope can be
44.              used to share variables between layers. Note that scope will
45.              override name.
46.          name: A name for this layer (optional). Default: 'ShallowBottleneck'.
47. 
    
48.      References:
49.          - Deep Residual Learning for Image Recognition. Kaiming He, Xiangyu
50.              Zhang, Shaoqing Ren, Jian Sun. 2015.
51.          - Identity Mappings in Deep Residual Networks. Kaiming He, Xiangyu
52.              Zhang, Shaoqing Ren, Jian Sun. 2015.
53. 
    
54.      Links:
55.          - [http://arxiv.org/pdf/1512.03385v1.pdf]
56.              ([http://arxiv.org/pdf/1512.03385v1.pdf](http://arxiv.org/pdf/1512.03385v1.pdf))
57.          - [Identity Mappings in Deep Residual Networks]
58.              ([http://arxiv.org/pdf/1603.05027v2.pdf](http://arxiv.org/pdf/1603.05027v2.pdf))
59. 
    
60.      """
61.      resnet = incoming
62.      in_channels = incoming.get_shape().as_list()[-1]
63. 
    
64.      with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope:
65.          name = scope.name #TODO
66. 
    
67.          for i in range(nb_blocks):
68. 
    
69.              identity = resnet
70. 
    
71.              if not downsample:
72.                  downsample_strides = 1
73. 
    
74.              if batch_norm:
75.                  resnet = tflearn.batch_normalization(resnet)
76.              resnet = tflearn.activation(resnet, activation)
77. 
    
78.              resnet = conv_2d(resnet, out_channels, 3,
79.                               downsample_strides, 'same', 'linear',
80.                               bias, weights_init, bias_init,
81.                               regularizer, weight_decay, trainable,
82.                               restore)
83. 
    
84.              if batch_norm:
85.                  resnet = tflearn.batch_normalization(resnet)
86.              resnet = tflearn.activation(resnet, activation)
87. 
    
88.              resnet = conv_2d(resnet, out_channels, 3, 1, 'same',
89.                               'linear', bias, weights_init,
90.                               bias_init, regularizer, weight_decay,
91.                               trainable, restore)
92. 
    
93.              # Downsampling
94.              if downsample_strides > 1:
95.                  identity = tflearn.avg_pool_2d(identity, 1,
96.                                                 downsample_strides)
97. 
    
98.              # Projection to new dimension
99.              if in_channels != out_channels:
100.                  ch = (out_channels - in_channels)//2
101.                  identity = tf.pad(identity,
102.                                    [[0, 0], [0, 0], [0, 0], [ch, ch]])
103.                  in_channels = out_channels
104. 
     
105.              resnet = resnet + identity
106. 
     
107.      return resnet