训练期间未使用Tensorflow/keras、GPU

Question

我用的是Tensorflow内置的Keras。 我使用的是conda env和tensorflow-gpu==1.10.0 我有CUDA 9.0和CUDN7

我看到我的GPU在训练开始的几秒钟内上升到30%（我猜它会加载图像）。然后，当CPU以20-25%的速度运行时，它会回到1%或2%

我尝试使用CUDA_VISIBLE_DEVICES=-1，我的CPU上升到94%（所以tensorflow使用的是GPU…）

这是我的模型和我如何训练它：

import numpy as np
import pandas as pd
import time

import tensorflow
from tensorflow import keras

taille_image = (96,96)      
batch_size = 32 

model = keras.Sequential()

model.add(keras.layers.Conv2D(16, (3, 3), input_shape=(taille_image[0], taille_image[1], 3),padding = "same", activation="relu"))
model.add(keras.layers.Dropout(0.25))
model.add(keras.layers.MaxPool2D(pool_size=(2,2)))

model.add(keras.layers.Conv2D(32, (3, 3), padding = "same", activation="relu"))
model.add(keras.layers.Dropout(0.25))
model.add(keras.layers.MaxPool2D(pool_size=(2,2)))

model.add(keras.layers.Conv2D(64, (3, 3), padding = "same", activation="relu"))
model.add(keras.layers.Dropout(0.25))
model.add(keras.layers.MaxPool2D(pool_size=(2,2)))

model.add(keras.layers.Conv2D(128, (3, 3), padding = "same", activation="relu"))
model.add(keras.layers.Dropout(0.25))
model.add(keras.layers.MaxPool2D(pool_size=(2,2)))

model.add(keras.layers.Conv2D(256, (3, 3), padding = "same", activation="relu"))
model.add(keras.layers.Dropout(0.25))
model.add(keras.layers.MaxPool2D(pool_size=(2,2)))

model.add(keras.layers.Flatten())                
model.add(keras.layers.Dense(3,activation="softmax"))

model.compile(loss='categorical_crossentropy',
        optimizer='adam',
        metrics=['accuracy'])
#model.summary()


def Entrainer(texte, barre, nb_epochs): 


        train_datagen = keras.preprocessing.image.ImageDataGenerator(
        rescale=1./255,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True,
        vertical_flip=True
        )
        valid_datagen = keras.preprocessing.image.ImageDataGenerator(rescale=1./255)

        test_datagen = keras.preprocessing.image.ImageDataGenerator(rescale=1./255)

        train_generator = train_datagen.flow_from_directory(
                directory='Images traitées/train',
                batch_size=batch_size,
                target_size=taille_image,
                color_mode="rgb",
                class_mode="categorical",
                shuffle=True,
                seed=42
        )

        validation_generator = valid_datagen.flow_from_directory(
                directory='Images traitées/validation',
                batch_size=1,
                target_size=taille_image,
                color_mode="rgb",
                class_mode="categorical",
                shuffle=True,
                seed=42
        )

        test_generator = test_datagen.flow_from_directory(
                directory='Images traitées/test',
                batch_size=1,
                target_size=taille_image,
                color_mode="rgb",
                class_mode=None,
                shuffle=False,
                seed=42
        )
        NAME = "16,32,64,128,256-conv-{}-batchs-{}".format(batch_size,int(time.time()))
        tensorboard = keras.callbacks.TensorBoard(log_dir = 'Graph/{}'.format(NAME))
        step_size_train = train_generator.n//train_generator.batch_size
        step_size_valid = validation_generator.n//validation_generator.batch_size
        step_size_test = test_generator.n//test_generator.batch_size

        model.fit_generator(
                generator=train_generator,
                steps_per_epoch=step_size_train,
                validation_data=validation_generator,
                validation_steps = step_size_valid,
                epochs=nb_epochs,
                callbacks = [tensorboard]
        )

它返回：

^{pr2}$

它相当慢，当我检查GPU的性能时： GPU 1% usage during training