add dropout and fc layers and use moment optimizer

Author: hujianwu

conv.py

@@ -18,12 +18,12 @@ def __init__(self,keep):
     def forward(self,x):
         linear = x.reshape(-1)
         self.outs = np.random.choice(len(linear),size=int(len(linear)*(1-self.keep))) 
-        linear[outs] = 0
+        linear[self.outs] = 0
         return x
 
     def backward(self,dy):
         linear = dy.reshape(-1)
-        linear[outs] = 0
+        linear[self.outs] = 0
         return dy
 
     def apply_gradients(self,learning_rate):
@@ -35,6 +35,8 @@ def __init__(self,filter,stride,padding):
         self.bias = np.zeros(shape=[filter.shape[-1]],dtype=float)
         self.stride = stride
         self.padding = padding
+        self.m_filter = np.zeros_like(self.filter)
+        self.m_bias = np.zeros_like(self.bias)
     def forward(self,x):
         padding = self.padding
         stride = self.stride
@@ -72,9 +74,12 @@ def backward(self,dy):
                 dexpand[i*stride:i*stride+fh,j*stride:j*stride+fw,:] += dgarden[i,j,:,:,:]
         return dexpand[padding:-padding,padding:-padding,:]
     def apply_gradients(self,learning_rate):
-        self.filter -= self.dfilter*learning_rate
-        self.bias -= self.dbias*learning_rate
-
+        self.m_bias = self.m_bias*0.9 + self.dbias*0.1
+        self.m_filter = self.m_filter*0.9 + self.dfilter*0.1
+        self.filter -= self.m_filter*learning_rate
+        self.bias -= self.m_bias*learning_rate
+        # self.filter -= self.dfilter*learning_rate
+        # self.bias -= self.dbias*learning_rate
 if __name__ == '__main__':
     image = np.ones(shape=[48,64,3])
     filter = np.ones(shape=[5,5,3,7])

mnist_conv.py

@@ -14,21 +14,24 @@
 idx = range(batch_size)
 
 
-filter0 = np.random.standard_normal(size=[3,3,1,5])*0.1
-filter = np.random.standard_normal(size=[3,3,5,5])*0.1
-conv_layer0 = conv.conv2d(filter0,2,1)
+filter0 = np.random.standard_normal(size=[3,3,1,3])*0.1
+filter = np.random.standard_normal(size=[3,3,3,5])*0.1
+conv_layer0 = conv.conv2d(filter0,1,1)
 relu0 = conv.relu()
+dropout0 = conv.dropout(0.6)
 conv_layer = conv.conv2d(filter,2,1)
 relu = conv.relu()
 
-l1 = neural.layer(28*28/4/4*5,10,"sigmoid",weight_decay=0.001)
+hidden = 100
+l1 = neural.layer(28*28/1/4*5,hidden,"linear",weight_decay=0.0)
+l2 = neural.layer(hidden,10,"softmax",weight_decay=0.0)
 
 loop = size//batch_size
 
-epoch = 3
-learning_rate = 0.02
-for _ in range(epoch):
-    for _ in range(loop):
+epoch = 1
+learning_rate = 0.0015
+for k in range(epoch):
+    for j in range(loop):
         images,labels = train.next_batch(batch_size)
         l = np.zeros([batch_size,label_num])
         l[idx,[labels]] = 1
@@ -37,40 +40,44 @@
             label = l[i].reshape(-1,1)
             conv_out0 = conv_layer0.forward(data)
             reludata0 = relu0.forward(conv_out0)
-            conv_out = conv_layer.forward(reludata0)
+            dropoutdata0 = dropout0.forward(reludata0)
+            conv_out = conv_layer.forward(dropoutdata0)
             reludata = relu.forward(conv_out)
             linear = reludata.reshape(-1,1)
             l1_out = l1.forward(linear)
-            loss = neural.sigmoid_loss(l1_out,label)
+            l2_out = l2.forward(l1_out)
+            loss = neural.softmax_loss(l2_out,label)
             print 'loss is ' + str(loss)
-            dloss = neural.prime_sigmoid_loss(l1_out,label)
-            dlinear = l1.backward(dloss)
+            dloss = neural.prime_softmax_loss(l2_out,label)
+            dl2_out = l2.backward(dloss)
+            dlinear = l1.backward(dl2_out)
             dreluout = dlinear.reshape(reludata.shape)
             dconv_out = relu.backward(dreluout)
             ddata = conv_layer.backward(dconv_out)
-            drelu0 = relu0.backward(ddata)
+            ddropout0 = dropout0.backward(ddata)
+            drelu0 = relu0.backward(ddropout0)
             dconv_out0 = conv_layer0.backward(drelu0)
             conv_layer.apply_gradients(learning_rate)
             conv_layer0.apply_gradients(learning_rate)
             l1.apply_gradients(learning_rate)
-    learning_rate *= 0.3
-         
+        if (j != 0 and j % 400 == 0):
+            learning_rate *= 0.4
 
     correct = 0
     test_size = test.num_examples
     for i in range(test_size):
-            images,labels = test.next_batch(1)
-            data = images[0]
-            label = labels[0]
-            conv_out0 = conv_layer0.forward(data)
-            reludata0 = relu0.forward(conv_out0)
-            conv_out = conv_layer.forward(reludata0)
-            reludata = relu.forward(conv_out)
-            linear = conv_out.reshape(-1,1)
-            l1_out = l1.forward(linear)
-            # l2_out = l2.forward(l1_out)
-            n = np.argmax(l1_out,axis=0)
-            if(n[0] == label ):
-                correct += 1
+        images,labels = test.next_batch(1)
+        data = images[0]
+        label = labels[0]
+        conv_out0 = conv_layer0.forward(data)
+        reludata0 = relu0.forward(conv_out0)
+        conv_out = conv_layer.forward(reludata0)
+        reludata = relu.forward(conv_out)
+        linear = conv_out.reshape(-1,1)
+        l1_out = l1.forward(linear)
+        l2_out = l2.forward(l1_out)
+        n = np.argmax(l2_out,axis=0)
+        if(n[0] == label ):
+            correct += 1
     print "{} correct, rate is {}".format(correct,correct*1.0/test_size)
     # IPython.embed()

mnist_reader.py

@@ -135,6 +135,10 @@ def __init__(self,
       images = numpy.multiply(images, 1.0 / 255.0)
     self._images = images
     self._labels = labels
+    perm = numpy.arange(self._num_examples)
+    numpy.random.shuffle(perm)
+    self._images = self._images[perm]
+    self._labels = self._labels[perm]
     self._epochs_completed = 0
     self._index_in_epoch = 0
 

neural.py

@@ -32,18 +32,20 @@ def __init__(self,active,prime_active):
 
 activations = dict()
 activations["sigmoid"] = activation(sigmoid,prime_sigmoid)
-activations["liner"] = activation(nop,prime_nop)
+activations["linear"] = activation(nop,prime_nop)
 activations["softmax"] = activation(softmax,prime_softmax)
 activations["tanh"] = activation(tanh,prime_tanh)
 class layer:
-    def __init__(self,input_size,output_size,active,bias_rate=1.0,weight_decay=0.001):
+    def __init__(self,input_size,output_size,active,bias_rate=1.0,weight_decay=0.0001):
         a = activations[active]
         self.fn_activate = a.active
         self.fn_prime_activate = a.prime_active
         self.weights = np.random.randn(output_size,input_size)/10.0
         self.d_weights = np.zeros((output_size,input_size))
         self.bias = np.random.randn(output_size,1)
         self.d_bias = np.zeros((output_size,1))
+        self.m_weights = np.zeros_like(self.weights)
+        self.m_bias = np.zeros_like(self.bias)
         self.bias_rate = bias_rate
         self.weight_decay = weight_decay
     def forward(self,x):
@@ -57,14 +59,18 @@ def backward(self,y):
             self.dy = y
         else:
             self.dy = self.fn_prime_activate(self.out) * y
-        self.d_weights = np.matmul(self.dy,self.x.T)/self.batch_size
-        self.d_bias = np.sum(self.dy,axis=1,keepdims=1)/self.batch_size
+        self.d_weights = np.matmul(self.dy,self.x.T)
+        self.d_bias = np.sum(self.dy,axis=1,keepdims=1)
         self.dx = np.matmul(self.weights.T,self.dy)
 
         return self.dx
     def apply_gradients(self,learning_rate):
-        self.weights = self.weights - self.d_weights*learning_rate - self.weights*self.weight_decay
-        self.bias -= self.d_bias
+        self.m_weights = self.m_weights*0.9 + self.d_weights*0.1
+        self.m_bias = self.m_bias*0.9 + self.d_bias*0.1
+        self.weights = self.weights - self.m_weights*learning_rate - self.weights*self.weight_decay
+        self.bias -= self.m_bias*learning_rate
+        # self.weights = self.weights - self.d_weights*learning_rate - self.weights*self.weight_decay
+        # self.bias -= self.d_bias*learning_rate
 
 def square_loss(a,y):
     l = np.sum((a-y)*(a-y))/2