用于视频识别的Keras-NCP模型

Question

试图创建一个识别手语的模型。使用三维卷积层，最后使用NCP库中的LTC单元。但模型精度在0.1后没有提高。 并且模型的输入维度为 (N,30,128,128,1) = (nvideos,frames,h,w,channel)

但我不确定输入模型时输入的形状是否正确。 在向模型提供数据之前，我将其重新格式化为(NVideos,1,30,128,128,1)。 只有我以这种方式重塑它，它才会起作用。 为什么模型精度没有提高

ncp_wiring = kncp.wirings.NCP(
    inter_neurons=16,  # Number of inter neurons
    command_neurons=8,  # Number of command neurons
    motor_neurons=len(exist_labels),  # Number of motor neurons
    sensory_fanout=4,  # How many outgoing synapses has each sensory neuron
    inter_fanout=3,  # How many outgoing synapses has each inter neuron
    recurrent_command_synapses=8,  # Now many recurrent synapses are in the
    # command neuron layer
    motor_fanin=4,  # How many incoming synapses has each motor neuron
)
ncp_cell = kncp.LTCCell(
    ncp_wiring,
    initialization_ranges={
        # Overwrite some of the initialization ranges
        "w": (0.2, 2.0),
    },
)

class Model:
    
    def __init__(self,load_name=None):
        if load_name != None:
            self.model = keras.models.load(load_name)       
        else:
            self.model = Sequential()
            sample_shape = (1,30,128,128,1);
            self.model.add(InputLayer(input_shape=sample_shape))
            self.model.add(TimeDistributed(Conv3D(32,kernel_size=(3,3,3),activation ='relu', kernel_initializer='he_uniform')))
            self.model.add(TimeDistributed(MaxPooling3D(pool_size=(2, 2, 2))))
            self.model.add(BatchNormalization(center=True, scale=True))
            self.model.add(Dropout(0.3))
            self.model.add(TimeDistributed(Conv3D(64, kernel_size=(3,3, 3), activation='relu', kernel_initializer='he_uniform')))
            self.model.add(TimeDistributed(MaxPooling3D(pool_size=(2, 2, 2))))
            self.model.add(BatchNormalization(center=True, scale=True))
            self.model.add(Dropout(0.3))
            self.model.add(TimeDistributed(Flatten()))
            self.model.add(TimeDistributed(Dense(16, activation='relu', kernel_initializer='he_uniform')))
            self.model.add(RNN(ncp_cell))
            self.model.add(keras.layers.Activation("softmax")),
            self.model.compile(loss='sparse_categorical_crossentropy',
                        optimizer = keras.optimizers.Adam(lr=0.01),
                        metrics=['accuracy'])
            self.model.build()
            self.model.summary() 
    
    def train(self,X_train,targets_train,batch,epochs,validation_split):
        history = self.model.fit(X_train, targets_train,
            batch_size=batch,
            epochs=epochs,
            verbose=1,
            validation_split=validation_split)
        print(history.history.keys())
        return [history.history['accuracy'],history.history['loss'],
                history.history['val_accuracy'],history.history['val_loss']]

    def save(self,name):
        self.model.save(name)