diff --git a/EmoPy/models/gcp-cnn.h5 b/EmoPy/models/gcp-cnn.h5 new file mode 100644 index 0000000..869faf8 Binary files /dev/null and b/EmoPy/models/gcp-cnn.h5 differ diff --git a/EmoPy/src/customLayer.py b/EmoPy/src/customLayer.py new file mode 100644 index 0000000..cbccc39 --- /dev/null +++ b/EmoPy/src/customLayer.py @@ -0,0 +1,85 @@ +from keras.engine import Layer +from keras.layers import Dense, Flatten, GlobalAveragePooling2D, Conv2D, ConvLSTM2D, Conv3D, MaxPooling2D, Dropout, \ + MaxPooling3D, K + +class SliceLayer(Layer): + def __init__(self, start=0, items=8, **kwargs): + super(SliceLayer, self).__init__(**kwargs) + self.start = start + self.items = items + self.trainable = False + + def build(self, input_shape): + super(SliceLayer, self).build(input_shape) + + def call(self, x): + return x[:, :, :, self.start: self.start + self.items] + + def compute_output_shape(self, input_shape): + height, width, in_channels = input_shape[1:] + return input_shape[0], height, width, self.items + + def get_config(self): + config = {'start': self.start, 'items': self.items} + base_config = super(SliceLayer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class ChannelShuffle(Layer): + def __init__(self, groups=None, groups_factor=8, **kwargs): + super(ChannelShuffle, self).__init__(**kwargs) + self.groups = groups + self.groups_factor = groups_factor + self.trainable = False + + def build(self, input_shape): + super(ChannelShuffle, self).build(input_shape) + + def call(self, x): + height, width, in_channels = x.shape.as_list()[1:] + + if self.groups is None: + if in_channels % self.groups_factor: + raise ValueError("%s %% %s" % (in_channels, self.groups_factor)) + + self.groups = in_channels // self.groups_factor + + channels_per_group = in_channels // self.groups + + x = K.reshape(x, [-1, height, width, self.groups, channels_per_group]) + x = K.permute_dimensions(x, (0, 1, 2, 4, 3)) # transpose + x = K.reshape(x, [-1, height, width, in_channels]) + + return x + + def compute_output_shape(self, input_shape): + return input_shape + + def get_config(self): + config = {'groups': self.groups, 'groups_factor': self.groups_factor} + base_config = super(ChannelShuffle, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class PadZeros(Layer): + def __init__(self, diff, **kwargs): + super(PadZeros, self).__init__(**kwargs) + self.diff = diff + self.trainable = False + + def build(self, input_shape): + # Create a trainable weight variable for this layer. + super(PadZeros, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, x): + import tensorflow as tf + return tf.pad(x, ((0, 0), (0, 0), (0, 0), (0, self.diff)), mode='CONSTANT') + + def compute_output_shape(self, input_shape): + batch, b_width, b_height, b_channels = input_shape + return batch, b_width, b_height, b_channels + self.diff + + def get_config(self): + config = {'diff': self.diff} + base_config = super(PadZeros, self).get_config() + return dict(list(base_config.items()) + list(config.items())) \ No newline at end of file diff --git a/EmoPy/src/neuralnets.py b/EmoPy/src/neuralnets.py index e4a218f..320235d 100644 --- a/EmoPy/src/neuralnets.py +++ b/EmoPy/src/neuralnets.py @@ -1,20 +1,26 @@ +import keras from keras.applications.inception_v3 import InceptionV3 from keras.applications.xception import Xception from keras.applications.vgg16 import VGG16 from keras.applications.vgg19 import VGG19 from keras.applications.resnet50 import ResNet50 from keras.callbacks import ReduceLROnPlateau, EarlyStopping +from keras.engine import Layer +from keras.engine.saving import load_model from keras.layers import Dense, Flatten, GlobalAveragePooling2D, Conv2D, ConvLSTM2D, Conv3D, MaxPooling2D, Dropout, \ - MaxPooling3D + MaxPooling3D, K from keras.layers.normalization import BatchNormalization from keras.losses import categorical_crossentropy from keras.models import Model, Sequential from keras.regularizers import l2 -from keras.optimizers import Adam +from keras.optimizers import Adam, Adamax from keras.utils import plot_model import json from EmoPy.src.callback import PlotLosses +from keras_preprocessing.image import ImageDataGenerator + +from EmoPy.EmoPy.src.customLayer import SliceLayer, ChannelShuffle, PadZeros class _FERNeuralNet(object): @@ -419,3 +425,55 @@ def fit(self, image_data, labels, validation_split, epochs=50): self.model.compile(optimizer="RMSProp", loss="cosine_proximity", metrics=["accuracy"]) self.model.fit(image_data, labels, epochs=epochs, validation_split=validation_split, callbacks=[ReduceLROnPlateau(), EarlyStopping(patience=3)]) + +class CGP_CNN(_FERNeuralNet): + """ + A Convolutional Neural Network model which was automatically found using a Cartesian Genetic Programming library. + https://github.com/scheckmedia/cgp-cnn-design + + :param emotion_map: dict of target emotion label keys with int values corresponding to the index of the emotion probability in the prediction output array + :param verbose: if true, will print out extra process information + """ + + def __init__(self,emotion_map,verbose=False): + self.verbose = verbose + super().__init__(emotion_map) + + def _init_model(self): + """ + Loads the network from the h5 file. + """ + model = load_model("EmoPy/models/gcp-cnn.h5", custom_objects={'SliceLayer': SliceLayer, 'ChannelShuffle': ChannelShuffle, + 'PadZeros': PadZeros}) + if self.verbose: + model.summary() + self.model = model + + def fit(self, x_train, y_train): + """ + Trains the neural net on the data provided. + + :param x_train: Train Data in the format: [:,48,48,1] + :param y_train: Label Data in the format: [:,7] + """ + datagen = ImageDataGenerator( + width_shift_range=0.1, + height_shift_range=0.1, + rotation_range=1, + horizontal_flip=True) + callback_LR = keras.callbacks.ReduceLROnPlateau( + monitor="val_acc", + factor=0.75, + patience=10, + verbose=1, + mode='auto', + min_delta=0.00001, + min_lr=0.0000001) + callback_LR.set_model(self.model) + callbacks = [callback_LR] + self.model.compile(loss='categorical_crossentropy', + optimizer=Adamax(lr=0.001), + metrics=['accuracy']) + self.model.fit_generator(datagen.flow(x_train, y_train, batch_size=32), + steps_per_epoch=(x_train.shape[0] // 32) + 1, verbose=1 if self.verbose else 0, epochs=200, callbacks=callbacks) +