How to create a 2-layer neural network using TensorFlow and python according to MNIST

Question

I am new to machine learning, and I am following the tenorflow tutorial to create simple neural networks that learn MNIST data.

I built a single layer network (after tutotial), the accuracy was about 0.92, which is suitable for me. But then I added another layer, the accuracy was reduced to 0.113, which is very bad.

The following is the relationship between the two layers:

import tensorflow as tf
x = tf.placeholder(tf.float32, [None, 784])

#layer 1
W1 = tf.Variable(tf.zeros([784, 100]))
b1 = tf.Variable(tf.zeros([100]))
y1 = tf.nn.softmax(tf.matmul(x, W1) + b1)

#layer 2
W2 = tf.Variable(tf.zeros([100, 10]))
b2 = tf.Variable(tf.zeros([10]))
y2 = tf.nn.softmax(tf.matmul(y1, W2) + b2)

#output
y = y2
y_ = tf.placeholder(tf.float32, [None, 10])

Is my structure beautiful? What is the reason for his failure? How to change my network?

Answer 1

Input of the second level is the softmaxoutput signal of the first level. You do not want to do this.

1. - tf.matmul(x, W1) + b1 0 (, , ), softmax 0. : , .

softmax ( softmax , ), .

; :

import tensorflow as tf
x = tf.placeholder(tf.float32, [None, 784])

#layer 1
W1 = tf.Variable(tf.zeros([784, 100]))
b1 = tf.Variable(tf.zeros([100]))
y1 = tf.matmul(x, W1) + b1 #remove softmax

#layer 2
W2 = tf.Variable(tf.zeros([100, 10]))
b2 = tf.Variable(tf.zeros([10]))
y2 = tf.nn.softmax(tf.matmul(y1, W2) + b2)

#output
y = y2
y_ = tf.placeholder(tf.float32, [None, 10])

Answer 2

​​ , nan y. , nessuno, , .

sigmoid 1, ! relu , initiate W1 W2 , 0.1135. relu sigmoid, W1 W2.

import tensorflow as tf

x = tf.placeholder(tf.float32, [None, 784])

# layer 1
with tf.variable_scope('layer1'):
    W1 = tf.get_variable('w1',[784,200],
                         initializer=tf.random_normal_initializer())
    b1 = tf.get_variable('b1',[1,],
                         initializer=tf.constant_initializer(0.0))
    y1 = tf.nn.sigmoid(tf.matmul(x, W1) + b1)
#   y1 = tf.nn.relu(tf.matmul(x, W1) + b1) # alternative choice for activation

# layer 2
with tf.variable_scope('layer2'):
    W2 = tf.get_variable('w2',[200,10],
                     initializer= tf.random_normal_nitializer())
    b2 = tf.get_variable('b2',[1,],
                         initializer=tf.constant_initializer(0.0))
    y2 = tf.nn.softmax(tf.matmul(y1, W2) + b2)

# output
y = y2
y_ = tf.placeholder(tf.float32, [None, 10])

, . 2 (), , , - , 1, , - (sth)*W2^T*W1^T, W1, W2 , , , , .

UPDATE

Quora Ofir .

Xavier . ( ) . "" 0, 0.

Answer 3

I tried to execute the above code snippets. Results below 90% were discarded, and I never felt confident that I had done what was in the comments above. Here is my complete code.

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
x = tf.placeholder(tf.float32, [None, 784])

#layer 1
W1 = tf.get_variable('w1', [784, 100], initializer=tf.random_normal_initializer())
b1 = tf.get_variable('b1', [1,], initializer=tf.random_normal_initializer())
y1 = tf.nn.sigmoid(tf.matmul(x, W1) + b1) 

#layer 2
W2 = tf.get_variable('w2',[100,10], initializer= 
tf.random_normal_initializer())
b2 = tf.get_variable('b2',[1,], initializer=tf.random_normal_initializer())
y2 = tf.nn.softmax(tf.matmul(y1, W2) + b2)

#output
y = y2
y_ = tf.placeholder(tf.float32, [None, 10])

cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), 
reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.2).minimize(cross_entropy)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()

for _ in range(10000):
  batch_xs, batch_ys = mnist.train.next_batch(100)
  sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})

correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: 
mnist.test.labels}))

By changing 10000 → 200000, I reached 95.5%.