Keras freezes when using ImageDataGenerator

Question

For some reason, when I use ImageDataGenerator with keras, it freezes when I start fitting. I get the following output. It just hangs on the Epoch 1/5 line

 Using Theano backend. Using gpu device 0: GeForce GTX TITAN (CNMeM is disabled, cuDNN not available) Loading Data Compiling Model Fitting Data Epoch 1/5 

This shows that one of my processor cores is 100% working, so something is happening on the processor, although it should use my GPU to match the data. The code below works if I comment on fit_generator and use the fit function.

 import os os.environ["THEANO_FLAGS"] = "mode=FAST_RUN,device=gpu,floatX=float32" import minst_loader import matplotlib.pyplot as plt import numpy as np from scipy.misc import imrotate import random from keras.datasets import cifar10 np.set_printoptions(suppress = True) print('Loading Data') x_train, y_train = (minst_loader.load_images('/home/chase/Desktop/MINST/train-images.idx3-ubyte'), \ minst_loader.load_labels('/home/chase/Desktop/MINST/train-labels.idx1-ubyte')) x_test, y_test = (minst_loader.load_images('/home/chase/Desktop/MINST/t10k-images.idx3-ubyte'), \ minst_loader.load_labels('/home/chase/Desktop/MINST/t10k-labels.idx1-ubyte')) for i in range(len(y_train)): v = np.zeros(10) v[y_train[i]] = 1 y_train[i] = v # for j in range(8): # x = imrotate(x_train[i], random.random() * 20) # x_train.append(x) # y_train.append(y_train[i]) for i in range(len(y_test)): v = np.zeros(10) v[y_test[i]] = 1 y_test[i] = v x_train = np.array(x_train) y_train = np.array(y_train) x_test = np.array(x_test) y_test = np.array(y_test) from keras.models import Sequential from keras.layers.core import Dense, Dropout, Flatten from keras.layers.noise import GaussianNoise from keras.preprocessing.image import ImageDataGenerator from keras.optimizers import SGD from keras.callbacks import ProgbarLogger, Callback datagen = ImageDataGenerator(rotation_range = 20, dim_ordering = 'tf') model = Sequential() model.add(Flatten(input_shape = (28, 28))) model.add(Dense(200, activation = 'tanh')) model.add(Dropout(0.5)) model.add(Dense(20, activation = 'tanh')) model.add(Dense(10, activation = 'softmax')) print('Compiling Model') sgd = SGD(lr = 0.01, decay = 0.1, momentum = 0.9, nesterov = True) model.compile(loss = 'categorical_crossentropy', optimizer = sgd) print('Fitting Data') #model.fit(x_train, y_train, batch_size = 128, nb_epoch = 400, validation_data = (x_test, y_test)) model.fit_generator(datagen.flow(x_train, y_train), samples_per_epoch = len(x_train), nb_epoch = 5) def max_index(lst): mi = 0 for i in range(1, len(lst)): mi = i if lst[i] > lst[mi] else mi return mi result = model.predict(x_test) correct = 0 for y, yt in zip(result, y_test): correct += max_index(y) == max_index(yt) print(correct / len(y_test)) 

Also here is my MINST bootloader if someone wants to run it ...

 import struct import numpy as np import matplotlib.pyplot as plt def load_images(images_file): data = None with open(images_file, 'rb') as f: data = f.read() mn, n, h, w = struct.unpack('>4I', data[0:16]) assert(mn == 2051) data = data[16:] images = [] for i in range(n): img = np.array([float(b) for b in data[w * h * i:w * h * (i + 1)]]) img /= 255.0 img = np.reshape(img, (w, h)) images.append(img) return images def load_labels(labels_file): data = None with open(labels_file, 'rb') as f: data = f.read() mn, n = struct.unpack('>2I', data[0:8]) assert(mn == 2049) return [int(b) for b in data[8:]] 

Answer 1

Try verbose=2 as a parameter in fit_generator()