TensorFlow 机器学习秘籍第二版：6~8（4）

# Plot loss over time 
plt.plot(i_data, train_loss, 'k-', label='Train Loss') 
plt.plot(i_data, test_loss, 'r--', label='Test Loss', linewidth=4) 
plt.title('Cross Entropy Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('Cross Entropy Loss') 
plt.legend(loc='upper right') 
plt.show() 
# Plot train and test accuracy 
plt.plot(i_data, train_acc, 'k-', label='Train Set Accuracy') 
plt.plot(i_data, test_acc, 'r--', label='Test Set Accuracy', linewidth=4) 
plt.title('Train and Test Accuracy') 
plt.xlabel('Generation') 
plt.ylabel('Accuracy') 
plt.legend(loc='lower right') 
plt.show()

import tensorflow as tf 
import matplotlib.pyplot as plt 
import numpy as np 
import random 
import os 
import pickle 
import string 
import requests 
import collections 
import io 
import tarfile 
import urllib.request 
import text_helpers 
from nltk.corpus import stopwords 
sess = tf.Session()

texts, target = text_helpers.load_movie_data()

batch_size = 500 
vocabulary_size = 7500 
generations = 100000 
model_learning_rate = 0.001 
embedding_size = 200   # Word embedding size 
doc_embedding_size = 100   # Document embedding size 
concatenated_size = embedding_size + doc_embedding_size 
num_sampled = int(batch_size/2) 
window_size = 3       # How many words to consider to the left. 
# Add checkpoints to training 
save_embeddings_every = 5000 
print_valid_every = 5000 
print_loss_every = 100 
# Declare stop words 
stops = stopwords.words('english') 
# We pick a few test words. 
valid_words = ['love', 'hate', 'happy', 'sad', 'man', 'woman']

texts = text_helpers.normalize_text(texts, stops)
# Texts must contain at least as much as the prior window size
target = [target[ix] for ix, x in enumerate(texts) if len(x.split()) > window_size]
texts = [x for x in texts if len(x.split()) > window_size]
assert(len(target)==len(texts))

word_dictionary = text_helpers.build_dictionary(texts, vocabulary_size) 
word_dictionary_rev = dict(zip(word_dictionary.values(), word_dictionary.keys())) 
text_data = text_helpers.text_to_numbers(texts, word_dictionary) 
# Get validation word keys 
valid_examples = [word_dictionary[x] for x in valid_words]

embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)) 
doc_embeddings = tf.Variable(tf.random_uniform([len(texts), doc_embedding_size], -1.0, 1.0)) 
# NCE loss parameters 
nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size, concatenated_size], 
                                               stddev=1.0 / np.sqrt(concatenated_size))) 
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

x_inputs = tf.placeholder(tf.int32, shape=[None, window_size + 1]) 
y_target = tf.placeholder(tf.int32, shape=[None, 1]) 
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

embed = tf.zeros([batch_size, embedding_size]) 
for element in range(window_size): 
    embed += tf.nn.embedding_lookup(embeddings, x_inputs[:, element]) 
doc_indices = tf.slice(x_inputs, [0,window_size],[batch_size,1]) 
doc_embed = tf.nn.embedding_lookup(doc_embeddings,doc_indices) 
# concatenate embeddings 
final_embed = tf.concat(axis=1, values=)

loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weights, 
                                     biases=nce_biases, 
                                     labels=y_target,
                                     inputs=final_embed,
                                     num_sampled=num_sampled, 
                                     num_classes=vocabulary_size))
# Create optimizer 
optimizer =  
 tf.train.GradientDescentOptimizer(learning_rate=model_learning_rate) 
train_step = optimizer.minimize(loss)

norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1,  
keep_dims=True)) 
normalized_embeddings = embeddings / norm 
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings,  
valid_dataset) 
similarity = tf.matmul(valid_embeddings, normalized_embeddings,  
transpose_b=True)

saver = tf.train.Saver({"embeddings": embeddings, "doc_embeddings":  
doc_embeddings}) 
init = tf.global_variables_initializer() 
sess.run(init) 
loss_vec = [] 
loss_x_vec = [] 
for i in range(generations): 
    batch_inputs, batch_labels = text_helpers.generate_batch_data(text_data, batch_size, 
                                                                  window_size, method='doc2vec') 
    feed_dict = {x_inputs : batch_inputs, y_target : batch_labels} 
    # Run the train step 
    sess.run(train_step, feed_dict=feed_dict) 
    # Return the loss 
    if (i+1) % print_loss_every == 0: 
        loss_val = sess.run(loss, feed_dict=feed_dict) 
        loss_vec.append(loss_val) 
        loss_x_vec.append(i+1) 
        print('Loss at step {} : {}'.format(i+1, loss_val)) 
    # Validation: Print some random words and top 5 related words 
    if (i+1) % print_valid_every == 0: 
        sim = sess.run(similarity, feed_dict=feed_dict) 
        for j in range(len(valid_words)): 
            valid_word = word_dictionary_rev[valid_examples[j]] 
            top_k = 5 # number of nearest neighbors 
            nearest = (-sim[j, :]).argsort()[1:top_k+1] 
            log_str = "Nearest to {}:".format(valid_word) 
            for k in range(top_k): 
                close_word = word_dictionary_rev[nearest[k]] 
                log_str = '{} {},'.format(log_str, close_word) 
            print(log_str) 
    # Save dictionary + embeddings 
    if (i+1) % save_embeddings_every == 0: 
        # Save vocabulary dictionary 
        with open(os.path.join(data_folder_name,'movie_vocab.pkl'), 'wb') as f: 
            pickle.dump(word_dictionary, f) 
        # Save embeddings 
        model_checkpoint_path = os.path.join(os.getcwd(),data_folder_name,'doc2vec_movie_embeddings.ckpt') 
        save_path = saver.save(sess, model_checkpoint_path) 
        print('Model saved in file: {}'.format(save_path))

Loss at step 100 : 126.176816940307617 
Loss at step 200 : 89.608322143554688
... 
Loss at step 99900 : 17.733346939086914 
Loss at step 100000 : 17.384489059448242 
Nearest to love: ride, with, by, its, start, 
Nearest to hate: redundant, snapshot, from, performances, extravagant, 
Nearest to happy: queen, chaos, them, succumb, elegance, 
Nearest to sad: terms, pity, chord, wallet, morality, 
Nearest to man: of, teen, an, our, physical, 
Nearest to woman: innocuous, scenes, prove, except, lady, 
Model saved in file: /.../temp/doc2vec_movie_embeddings.ckpt

max_words = 20 # maximum review word length 
logistic_batch_size = 500 # training batch size

train_indices = np.sort(np.random.choice(len(target),  
round(0.8*len(target)), replace=False)) 
test_indices = np.sort(np.array(list(set(range(len(target))) -  
set(train_indices)))) 
texts_train = [x for ix, x in enumerate(texts) if ix in train_indices] 
texts_test = [x for ix, x in enumerate(texts) if ix in test_indices] 
target_train = np.array([x for ix, x in enumerate(target) if ix in train_indices]) 
target_test = np.array([x for ix, x in enumerate(target) if ix in test_indices])

text_data_train = np.array(text_helpers.text_to_numbers(texts_train, word_dictionary)) text_data_test = np.array(text_helpers.text_to_numbers(texts_test, word_dictionary)) # Pad/crop movie reviews to specific length text_data_train = np.array([x[0:max_words] for x in [y+[0]*max_words for y in text_data_train]]) text_data_test = np.array([x[0:max_words] for x in [y+[0]*max_words for y in text_data_test]])

# Define Logistic placeholders 
log_x_inputs = tf.placeholder(tf.int32, shape=[None, max_words + 1]) 
log_y_target = tf.placeholder(tf.int32, shape=[None, 1]) 
A = tf.Variable(tf.random_normal(shape=[concatenated_size,1])) 
b = tf.Variable(tf.random_normal(shape=[1,1])) 
# Declare logistic model (sigmoid in loss function) 
model_output = tf.add(tf.matmul(log_final_embed, A), b) 
# Declare loss function (Cross Entropy loss) 
logistic_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=model_output,  
labels=tf.cast(log_y_target, tf.float32)))

# Add together element embeddings in window: 
log_embed = tf.zeros([logistic_batch_size, embedding_size]) 
for element in range(max_words): 
    log_embed += tf.nn.embedding_lookup(embeddings, log_x_inputs[:, element]) 
log_doc_indices = tf.slice(log_x_inputs, [0,max_words],[logistic_batch_size,1]) 
log_doc_embed = tf.nn.embedding_lookup(doc_embeddings,log_doc_indices) 
# concatenate embeddings 
log_final_embed = tf.concat(1, [log_embed, tf.squeeze(log_doc_embed)])

prediction = tf.round(tf.sigmoid(model_output)) 
predictions_correct = tf.cast(tf.equal(prediction, tf.cast(log_y_target, tf.float32)), tf.float32) 
accuracy = tf.reduce_mean(predictions_correct) 
# Declare optimizer 
logistic_opt = tf.train.GradientDescentOptimizer(learning_rate=0.01) 
logistic_train_step = logistic_opt.minimize(logistic_loss, var_list=[A, b]) 
# Intitialize Variables 
init = tf.global_variables_initializer() 
sess.run(init)

train_loss = [] 
test_loss = [] 
train_acc = [] 
test_acc = [] 
i_data = [] 
for i in range(10000): 
    rand_index = np.random.choice(text_data_train.shape[0], size=logistic_batch_size) 
    rand_x = text_data_train[rand_index] 
    # Append review index at the end of text data 
    rand_x_doc_indices = train_indices[rand_index] 
    rand_x = np.hstack((rand_x, np.transpose([rand_x_doc_indices]))) 
    rand_y = np.transpose([target_train[rand_index]]) 
    feed_dict = {log_x_inputs : rand_x, log_y_target : rand_y} 
    sess.run(logistic_train_step, feed_dict=feed_dict) 
    # Only record loss and accuracy every 100 generations 
    if (i+1)%100==0: 
        rand_index_test = np.random.choice(text_data_test.shape[0], size=logistic_batch_size) 
        rand_x_test = text_data_test[rand_index_test] 
        # Append review index at the end of text data 
        rand_x_doc_indices_test = test_indices[rand_index_test] 
        rand_x_test = np.hstack((rand_x_test, np.transpose([rand_x_doc_indices_test]))) 
        rand_y_test = np.transpose([target_test[rand_index_test]]) 
        test_feed_dict = {log_x_inputs: rand_x_test, log_y_target: rand_y_test} 
        i_data.append(i+1) 
        train_loss_temp = sess.run(logistic_loss, feed_dict=feed_dict) 
        train_loss.append(train_loss_temp) 
        test_loss_temp = sess.run(logistic_loss, feed_dict=test_feed_dict) 
        test_loss.append(test_loss_temp) 
        train_acc_temp = sess.run(accuracy, feed_dict=feed_dict) 
        train_acc.append(train_acc_temp) 
        test_acc_temp = sess.run(accuracy, feed_dict=test_feed_dict) 
        test_acc.append(test_acc_temp) 
    if (i+1)%500==0: 
        acc_and_loss = [i+1, train_loss_temp, test_loss_temp, train_acc_temp, test_acc_temp] 
        acc_and_loss = [np.round(x,2) for x in acc_and_loss] 
        print('Generation # {}. Train Loss (Test Loss): {:.2f} ({:.2f}). Train Acc (Test Acc): {:.2f} ({:.2f})'.format(*acc_and_loss))

Generation # 500\. Train Loss (Test Loss): 5.62 (7.45). Train Acc (Test Acc): 0.52 (0.48) Generation # 10000\. Train Loss (Test Loss): 2.35 (2.51). Train Acc (Test Acc): 0.59 (0.58)

def generate_batch_data(sentences, batch_size, window_size, method='skip_gram'): 
    # Fill up data batch 
    batch_data = [] 
    label_data = [] 
    while len(batch_data) < batch_size: 
        # select random sentence to start 
        rand_sentence_ix = int(np.random.choice(len(sentences), size=1)) 
        rand_sentence = sentences[rand_sentence_ix] 
        # Generate consecutive windows to look at 
        window_sequences = [rand_sentence[max((ix-window_size),0):(ix+window_size+1)] for ix, x in enumerate(rand_sentence)] 
        # Denote which element of each window is the center word of interest 
        label_indices = [ix if ix<window_size else window_size for ix,x in enumerate(window_sequences)] 
        # Pull out center word of interest for each window and create a tuple for each window 
        if method=='skip_gram': 
            ... 
        elif method=='cbow': 
            ... 
        elif method=='doc2vec': 
            # For doc2vec we keep LHS window only to predict target word 
            batch_and_labels = [(rand_sentence[i:i+window_size], rand_sentence[i+window_size]) for i in range(0, len(rand_sentence)-window_size)] 
            batch, labels = [list(x) for x in zip(*batch_and_labels)] 
            # Add document index to batch!! Remember that we must extract the last index in batch for the doc-index 
            batch = [x + [rand_sentence_ix] for x in batch] 
        else: 
            raise ValueError('Method {} not implmented yet.'.format(method)) 
        # extract batch and labels 
        batch_data.extend(batch[:batch_size]) 
        label_data.extend(labels[:batch_size]) 
    # Trim batch and label at the end 
    batch_data = batch_data[:batch_size] 
    label_data = label_data[:batch_size] 
    # Convert to numpy array 
    batch_data = np.array(batch_data) 
    label_data = np.transpose(np.array([label_data])) 
    return batch_data, label_data

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from tensorflow.python.framework import ops
ops.reset_default_graph()
sess = tf.Session()

data_dir = 'temp' 
mnist = input_data.read_data_sets(data_dir, one_hot=False)
train_xdata = np.array([np.reshape(x, (28,28)) for x in mnist.train.images]) 
test_xdata = np.array([np.reshape(x, (28,28)) for x in mnist.test.images]) 
train_labels = mnist.train.labels 
test_labels = mnist.test.labels

batch_size = 100 
learning_rate = 0.005 
evaluation_size = 500 
image_width = train_xdata[0].shape[0] 
image_height = train_xdata[0].shape[1] 
target_size = max(train_labels) + 1 
num_channels = 1 
generations = 500 
eval_every = 5 
conv1_features = 25 
conv2_features = 50 
max_pool_size1 = 2 
max_pool_size2 = 2 
fully_connected_size1 = 100

x_input_shape = (batch_size, image_width, image_height, num_channels) 
x_input = tf.placeholder(tf.float32, shape=x_input_shape) 
y_target = tf.placeholder(tf.int32, shape=(batch_size)) 
eval_input_shape = (evaluation_size, image_width, image_height, num_channels) 
eval_input = tf.placeholder(tf.float32, shape=eval_input_shape) 
eval_target = tf.placeholder(tf.int32, shape=(evaluation_size))

conv1_weight = tf.Variable(tf.truncated_normal([4, 4, num_channels, conv1_features], stddev=0.1, dtype=tf.float32)) 
conv1_bias = tf.Variable(tf.zeros([conv1_features],dtype=tf.float32)) 
conv2_weight = tf.Variable(tf.truncated_normal([4, 4, conv1_features, conv2_features], stddev=0.1, dtype=tf.float32)) 
conv2_bias = tf.Variable(tf.zeros([conv2_features],dtype=tf.float32))

resulting_width = image_width // (max_pool_size1 * max_pool_size2) 
resulting_height = image_height // (max_pool_size1 * max_pool_size2) 
full1_input_size = resulting_width * resulting_height*conv2_features 
full1_weight = tf.Variable(tf.truncated_normal([full1_input_size, fully_connected_size1], stddev=0.1, dtype=tf.float32)) 
full1_bias = tf.Variable(tf.truncated_normal([fully_connected_size1], stddev=0.1, dtype=tf.float32)) 
full2_weight = tf.Variable(tf.truncated_normal([fully_connected_size1, target_size], stddev=0.1, dtype=tf.float32)) 
full2_bias = tf.Variable(tf.truncated_normal([target_size], stddev=0.1, dtype=tf.float32))

def my_conv_net(input_data): 
   # First Conv-ReLU-MaxPool Layer 
    conv1 = tf.nn.conv2d(input_data, conv1_weight, strides=[1, 1, 1, 1], padding='SAME') 
    relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_bias)) 
    max_pool1 = tf.nn.max_pool(relu1, ksize=[1, max_pool_size1, max_pool_size1, 1], strides=[1, max_pool_size1, max_pool_size1, 1], padding='SAME') 
    # Second Conv-ReLU-MaxPool Layer 
    conv2 = tf.nn.conv2d(max_pool1, conv2_weight, strides=[1, 1, 1, 1], padding='SAME') 
    relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_bias)) 
    max_pool2 = tf.nn.max_pool(relu2, ksize=[1, max_pool_size2, max_pool_size2, 1], strides=[1, max_pool_size2, max_pool_size2, 1], padding='SAME') 
    # Transform Output into a 1xN layer for next fully connected layer 
    final_conv_shape = max_pool2.get_shape().as_list() 
    final_shape = final_conv_shape[1] * final_conv_shape[2] * final_conv_shape[3] 
    flat_output = tf.reshape(max_pool2, [final_conv_shape[0], final_shape]) 
    # First Fully Connected Layer 
    fully_connected1 = tf.nn.relu(tf.add(tf.matmul(flat_output, full1_weight), full1_bias)) 
    # Second Fully Connected Layer 
    final_model_output = tf.add(tf.matmul(fully_connected1, full2_weight), full2_bias) 
    return final_model_output

model_output = my_conv_net(x_input) 
test_model_output = my_conv_net(eval_input)

loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=model_output, labels=y_target))

prediction = tf.nn.softmax(model_output) 
test_prediction = tf.nn.softmax(test_model_output) 
# Create accuracy function 
def get_accuracy(logits, targets): 
    batch_predictions = np.argmax(logits, axis=1) 
    num_correct = np.sum(np.equal(batch_predictions, targets)) 
    return 100\. * num_correct/batch_predictions.shape[0]

my_optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9) 
train_step = my_optimizer.minimize(loss) 
# Initialize Variables 
init = tf.global_variables_initializer() 
sess.run(init)

train_loss = [] 
train_acc = [] 
test_acc = [] 
for i in range(generations): 
    rand_index = np.random.choice(len(train_xdata), size=batch_size) 
    rand_x = train_xdata[rand_index] 
    rand_x = np.expand_dims(rand_x, 3) 
    rand_y = train_labels[rand_index] 
    train_dict = {x_input: rand_x, y_target: rand_y} 
    sess.run(train_step, feed_dict=train_dict) 
    temp_train_loss, temp_train_preds = sess.run([loss, prediction], feed_dict=train_dict) 
    temp_train_acc = get_accuracy(temp_train_preds, rand_y) 
    if (i+1) % eval_every == 0: 
        eval_index = np.random.choice(len(test_xdata), size=evaluation_size) 
        eval_x = test_xdata[eval_index] 
        eval_x = np.expand_dims(eval_x, 3) 
        eval_y = test_labels[eval_index] 
        test_dict = {eval_input: eval_x, eval_target: eval_y} 
        test_preds = sess.run(test_prediction, feed_dict=test_dict) 
        temp_test_acc = get_accuracy(test_preds, eval_y) 
        # Record and print results 
        train_loss.append(temp_train_loss) 
        train_acc.append(temp_train_acc) 
        test_acc.append(temp_test_acc) 
        acc_and_loss = [(i+1), temp_train_loss, temp_train_acc, temp_test_acc] 
        acc_and_loss = [np.round(x,2) for x in acc_and_loss] 
        print('Generation # {}. Train Loss: {:.2f}. Train Acc (Test Acc): {:.2f} ({:.2f})'.format(*acc_and_loss))

Generation # 5\. Train Loss: 2.37\. Train Acc (Test Acc): 7.00 (9.80) 
Generation # 10\. Train Loss: 2.16\. Train Acc (Test Acc): 31.00 (22.00) 
Generation # 15\. Train Loss: 2.11\. Train Acc (Test Acc): 36.00 (35.20) 
...
Generation # 490\. Train Loss: 0.06\. Train Acc (Test Acc): 98.00 (97.40) 
Generation # 495\. Train Loss: 0.10\. Train Acc (Test Acc): 98.00 (95.40) 
Generation # 500\. Train Loss: 0.14\. Train Acc (Test Acc): 98.00 (96.00)

eval_indices = range(0, generations, eval_every) 
# Plot loss over time 
plt.plot(eval_indices, train_loss, 'k-') 
plt.title('Softmax Loss per Generation') 
plt.xlabel('Generation') 
plt.ylabel('Softmax Loss') 
plt.show() 
# Plot train and test accuracy 
plt.plot(eval_indices, train_acc, 'k-', label='Train Set Accuracy') 
plt.plot(eval_indices, test_acc, 'r--', label='Test Set Accuracy') 
plt.title('Train and Test Accuracy') 
plt.xlabel('Generation') 
plt.ylabel('Accuracy') 
plt.legend(loc='lower right') 
plt.show()

# Plot the 6 of the last batch results: 
actuals = rand_y[0:6] 
predictions = np.argmax(temp_train_preds,axis=1)[0:6] 
images = np.squeeze(rand_x[0:6]) 
Nrows = 2 
Ncols = 3 
for i in range(6): 
    plt.subplot(Nrows, Ncols, i+1) 
    plt.imshow(np.reshape(images[i], [28,28]), cmap='Greys_r') 
    plt.title('Actual: ' + str(actuals[i]) + ' Pred: ' + str(predictions[i]), fontsize=10) 
    frame = plt.gca() 
    frame.axes.get_xaxis().set_visible(False) 
    frame.axes.get_yaxis().set_visible(False)

import os 
import sys 
import tarfile 
import matplotlib.pyplot as plt 
import numpy as np 
import tensorflow as tf 
from six.moves import urllib 
sess = tf.Session()

batch_size = 128 
output_every = 50 
generations = 20000 
eval_every = 500 
image_height = 32 
image_width = 32 
crop_height = 24 
crop_width = 24 
num_channels = 3 
num_targets = 10 
data_dir = 'temp' 
extract_folder = 'cifar-10-batches-bin'

learning_rate = 0.1 
lr_decay = 0.9 
num_gens_to_wait = 250\.

image_vec_length = image_height * image_width * num_channels 
record_length = 1 + image_vec_length

data_dir = 'temp' 
if not os.path.exists(data_dir): 
    os.makedirs(data_dir) 
cifar10_url = 'http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' 
data_file = os.path.join(data_dir, 'cifar-10-binary.tar.gz') 
if not os.path.isfile(data_file): 
    # Download file 
    filepath, _ = urllib.request.urlretrieve(cifar10_url, data_file) 
    # Extract file 
    tarfile.open(filepath, 'r:gz').extractall(data_dir)

def read_cifar_files(filename_queue, distort_images = True): 
    reader = tf.FixedLengthRecordReader(record_bytes=record_length) 
    key, record_string = reader.read(filename_queue) 
    record_bytes = tf.decode_raw(record_string, tf.uint8) 
    # Extract label 
    image_label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32) 
    # Extract image 
    image_extracted = tf.reshape(tf.slice(record_bytes, [1], [image_vec_length]), [num_channels, image_height, image_width]) 
    # Reshape image 
    image_uint8image = tf.transpose(image_extracted, [1, 2, 0]) 
    reshaped_image = tf.cast(image_uint8image, tf.float32) 
    # Randomly Crop image 
    final_image = tf.image.resize_image_with_crop_or_pad(reshaped_image, crop_width, crop_height) 
    if distort_images: 
        # Randomly flip the image horizontally, change the brightness and contrast 
        final_image = tf.image.random_flip_left_right(final_image) 
        final_image = tf.image.random_brightness(final_image,max_delta=63) 
        final_image = tf.image.random_contrast(final_image,lower=0.2, upper=1.8) 
    # Normalize whitening 
    final_image = tf.image.per_image_standardization(final_image) 
    return final_image, image_label

def input_pipeline(batch_size, train_logical=True): 
    if train_logical: 
        files = [os.path.join(data_dir, extract_folder, 'data_batch_{}.bin'.format(i)) for i in range(1,6)] 
    else: 
        files = [os.path.join(data_dir, extract_folder, 'test_batch.bin')] 
    filename_queue = tf.train.string_input_producer(files) 
    image, label = read_cifar_files(filename_queue) 
    min_after_dequeue = 1000 
    capacity = min_after_dequeue + 3 * batch_size 
    example_batch, label_batch = tf.train.shuffle_batch([image, label], batch_size, capacity, min_after_dequeue) 
    return example_batch, label_batch

def cifar_cnn_model(input_images, batch_size, train_logical=True): 
    def truncated_normal_var(name, shape, dtype): 
        return tf.get_variable(name=name, shape=shape, dtype=dtype, initializer=tf.truncated_normal_initializer(stddev=0.05)) 
    def zero_var(name, shape, dtype): 
        return tf.get_variable(name=name, shape=shape, dtype=dtype, initializer=tf.constant_initializer(0.0))
    # First Convolutional Layer 
    with tf.variable_scope('conv1') as scope: 
        # Conv_kernel is 5x5 for all 3 colors and we will create 64 features 
        conv1_kernel = truncated_normal_var(name='conv_kernel1', shape=[5, 5, 3, 64], dtype=tf.float32) 
        # We convolve across the image with a stride size of 1 
        conv1 = tf.nn.conv2d(input_images, conv1_kernel, [1, 1, 1, 1], padding='SAME') 
        # Initialize and add the bias term 
        conv1_bias = zero_var(name='conv_bias1', shape=[64], dtype=tf.float32) 
        conv1_add_bias = tf.nn.bias_add(conv1, conv1_bias) 
        # ReLU element wise 
        relu_conv1 = tf.nn.relu(conv1_add_bias) 
    # Max Pooling 
    pool1 = tf.nn.max_pool(relu_conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],padding='SAME', name='pool_layer1') 
    # Local Response Normalization 
    norm1 = tf.nn.lrn(pool1, depth_radius=5, bias=2.0, alpha=1e-3, beta=0.75, name='norm1') 
    # Second Convolutional Layer 
    with tf.variable_scope('conv2') as scope: 
        # Conv kernel is 5x5, across all prior 64 features and we create 64 more features 
        conv2_kernel = truncated_normal_var(name='conv_kernel2', shape=[5, 5, 64, 64], dtype=tf.float32) 
        # Convolve filter across prior output with stride size of 1 
        conv2 = tf.nn.conv2d(norm1, conv2_kernel, [1, 1, 1, 1], padding='SAME') 
        # Initialize and add the bias 
        conv2_bias = zero_var(name='conv_bias2', shape=[64], dtype=tf.float32) 
        conv2_add_bias = tf.nn.bias_add(conv2, conv2_bias) 
        # ReLU element wise 
        relu_conv2 = tf.nn.relu(conv2_add_bias) 
    # Max Pooling 
    pool2 = tf.nn.max_pool(relu_conv2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool_layer2')     
     # Local Response Normalization (parameters from paper) 
    norm2 = tf.nn.lrn(pool2, depth_radius=5, bias=2.0, alpha=1e-3, beta=0.75, name='norm2') 
    # Reshape output into a single matrix for multiplication for the fully connected layers 
    reshaped_output = tf.reshape(norm2, [batch_size, -1]) 
    reshaped_dim = reshaped_output.get_shape()[1].value 
    # First Fully Connected Layer 
    with tf.variable_scope('full1') as scope: 
        # Fully connected layer will have 384 outputs. 
        full_weight1 = truncated_normal_var(name='full_mult1', shape=[reshaped_dim, 384], dtype=tf.float32) 
        full_bias1 = zero_var(name='full_bias1', shape=[384], dtype=tf.float32) 
        full_layer1 = tf.nn.relu(tf.add(tf.matmul(reshaped_output, full_weight1), full_bias1)) 
    # Second Fully Connected Layer 
    with tf.variable_scope('full2') as scope: 
        # Second fully connected layer has 192 outputs. 
        full_weight2 = truncated_normal_var(name='full_mult2', shape=[384, 192], dtype=tf.float32) 
        full_bias2 = zero_var(name='full_bias2', shape=[192], dtype=tf.float32) 
        full_layer2 = tf.nn.relu(tf.add(tf.matmul(full_layer1, full_weight2), full_bias2)) 
    # Final Fully Connected Layer -> 10 categories for output (num_targets) 
    with tf.variable_scope('full3') as scope: 
        # Final fully connected layer has 10 (num_targets) outputs. 
        full_weight3 = truncated_normal_var(name='full_mult3', shape=[192, num_targets], dtype=tf.float32) 
        full_bias3 =  zero_var(name='full_bias3', shape=[num_targets], dtype=tf.float32) 
        final_output = tf.add(tf.matmul(full_layer2, full_weight3), full_bias3) 
    return final_output

def cifar_loss(logits, targets): 
    # Get rid of extra dimensions and cast targets into integers 
    targets = tf.squeeze(tf.cast(targets, tf.int32)) 
    # Calculate cross entropy from logits and targets 
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets) 
    # Take the average loss across batch size 
    cross_entropy_mean = tf.reduce_mean(cross_entropy) 
    return cross_entropy_mean

def train_step(loss_value, generation_num): 
    # Our learning rate is an exponential decay (stepped down) 
    model_learning_rate = tf.train.exponential_decay(learning_rate, generation_num, num_gens_to_wait, lr_decay, staircase=True) 
    # Create optimizer 
    my_optimizer = tf.train.GradientDescentOptimizer(model_learning_rate) 
    # Initialize train step 
    train_step = my_optimizer.minimize(loss_value) 
    return train_step

def accuracy_of_batch(logits, targets): 
    # Make sure targets are integers and drop extra dimensions 
    targets = tf.squeeze(tf.cast(targets, tf.int32)) 
    # Get predicted values by finding which logit is the greatest 
    batch_predictions = tf.cast(tf.argmax(logits, 1), tf.int32) 
    # Check if they are equal across the batch 
    predicted_correctly = tf.equal(batch_predictions, targets) 
    # Average the 1's and 0's (True's and False's) across the batch size 
    accuracy = tf.reduce_mean(tf.cast(predicted_correctly, tf.float32)) 
    return accuracy

images, targets = input_pipeline(batch_size, train_logical=True) 
test_images, test_targets = input_pipeline(batch_size, train_logical=False)

with tf.variable_scope('model_definition') as scope: 
    # Declare the training network model 
    model_output = cifar_cnn_model(images, batch_size) 
    # Use same variables within scope 
    scope.reuse_variables() 
    # Declare test model output 
    test_output = cifar_cnn_model(test_images, batch_size)

loss = cifar_loss(model_output, targets) 
accuracy = accuracy_of_batch(test_output, test_targets) 
generation_num = tf.Variable(0, trainable=False) 
train_op = train_step(loss, generation_num)

init = tf.global_variables_initializer() 
sess.run(init) 
tf.train.start_queue_runners(sess=sess)

train_loss = [] 
test_accuracy = [] 
for i in range(generations): 
    _, loss_value = sess.run([train_op, loss]) 
    if (i+1) % output_every == 0: 
        train_loss.append(loss_value) 
        output = 'Generation {}: Loss = {:.5f}'.format((i+1), loss_value) 
        print(output) 
    if (i+1) % eval_every == 0: 
        [temp_accuracy] = sess.run([accuracy]) 
        test_accuracy.append(temp_accuracy) 
        acc_output = ' --- Test Accuracy= {:.2f}%.'.format(100\. * temp_accuracy) 
        print(acc_output)