TensorFlow image.decode_png在几次成功的培训后损坏

Question

我正在训练一个神经网络，并且已经成功地将所有的训练数据运行了好几个时期。然而，TensorFlow image.decode_png损坏的错误突然出现如下：

tensorflow.python.framework.errors_impl.InvalidArgumentError: Invalid PNG data, size 929733 [Op:DecodePng]

我再次检查了数据文件，它确实已损坏。但在我运行培训代码之前，数据是完整的，我只是通过以下代码读取数据：

import transformations
class VisualOdometryDataLoader:
    def __init__(self, datapath, height, width, batch_size, test=False, val=False, sequence_test='10'):
        self.base_path = datapath
        if test or val:
            self.sequences = [sequence_test] # 01, 03, 05, 09
        else:
            # self.sequences = ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21']
            # self.sequences = ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10']
            self.sequences = ['00', '02', '04', '06', '08', '10']#00, 02, 06, 08, 10

        self.size = 0
        self.sizes = []
        self.poses = self.load_poses()
        self.width = width
        self.height = height

        images_stacked, odometries = self.get_data()
        dataset = tf.data.Dataset.from_tensor_slices((images_stacked, odometries))

        if test:
            dataset = dataset.map(self.load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE)
            dataset = dataset.batch(batch_size)
        else:
#             dataset = dataset.shuffle(len(images_stacked))
            dataset = dataset.map(self.load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE)
            dataset = dataset.batch(batch_size)
            dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)

        self.dataset = dataset

    def decode_img(self, img):
        image = tf.image.decode_png(img, channels=3)
        image = image[..., ::-1]
        image = tf.image.convert_image_dtype(image, tf.float32)
        image = tf.image.resize(image, [self.height, self.width])
        return image

    def load_image(self, filename, odometry):
        try:
            img1 = tf.io.read_file(filename[0])
            img2 = tf.io.read_file(filename[1])
            img1 = self.decode_img(img1)
            img2 = self.decode_img(img2)
            img = tf.concat([img1, img2], -1)
            return img, odometry
        except Exception as e:
            print(filename)
            print('error '+filename+"\n"+str(e))
            traceback.print_exc()

    def load_poses(self):
        all_poses = []
        for sequence in self.sequences:
            with open(os.path.join(self.base_path, 'poses/', sequence + '.txt')) as f:
                poses = np.array([[float(x) for x in line.split()] for line in f], dtype=np.float32)
                all_poses.append(poses)

                self.size = self.size + len(poses)
                self.sizes.append(len(poses))
        return all_poses

    def get_image_paths(self, sequence, index):
        image_path = os.path.join(self.base_path, 'sequences', sequence, 'image_2', '%06d' % index + '.png')
        return image_path

    def isRotationMatrix(self, R):
        Rt = np.transpose(R)
        shouldBeIdentity = np.dot(Rt, R)
        I = np.identity(3, dtype=R.dtype)
        n = np.linalg.norm(I - shouldBeIdentity)
        return n < 1e-6

    def rotationMatrixToEulerAngles(self, R):
        assert (self.isRotationMatrix(R))
        sy = math.sqrt(R[0, 0] * R[0, 0] + R[1, 0] * R[1, 0])
        singular = sy < 1e-6

        if not singular:
            x = math.atan2(R[2, 1], R[2, 2])
            y = math.atan2(-R[2, 0], sy)
            z = math.atan2(R[1, 0], R[0, 0])
        else:
            x = math.atan2(-R[1, 2], R[1, 1])
            y = math.atan2(-R[2, 0], sy)
            z = 0
        return np.array([x, y, z], dtype=np.float32)

    def eulerAnglesToRotationMatrix(self, theta):
        R_x = np.array([[1, 0, 0],
                        [0, np.cos(theta[0]), -np.sin(theta[0])],
                        [0, np.sin(theta[0]), np.cos(theta[0])]
                        ])
        R_y = np.array([[np.cos(theta[1]), 0, np.sin(theta[1])],
                        [0, 1, 0],
                        [-np.sin(theta[1]), 0, np.cos(theta[1])]
                        ])
        R_z = np.array([[np.cos(theta[2]), -np.sin(theta[2]), 0],
                        [np.sin(theta[2]), np.cos(theta[2]), 0],
                        [0, 0, 1]
                        ])
        R = np.dot(R_z, np.dot(R_y, R_x))
        return R

    def matrix_rt(self, p):
        return np.vstack([np.reshape(p.astype(np.float32), (3, 4)), [[0., 0., 0., 1.]]])

    def get_data(self):
        images_paths = []
        odometries = []
        for index, sequence in enumerate(self.sequences):
            for i in range(self.sizes[index] - 1):
                images_paths.append([self.get_image_paths(sequence, i), self.get_image_paths(sequence, i + 1)])
                pose1 = self.matrix_rt(self.poses[index][i])
                pose2 = self.matrix_rt(self.poses[index][i + 1])
                pose2wrt1 = np.dot(np.linalg.inv(pose1), pose2)
                R = pose2wrt1[0:3, 0:3]
                t = pose2wrt1[0:3, 3]
#                 angles = self.rotationMatrixToEulerAngles(R)
                al, be, ga = transformations.euler_from_matrix(R, 'rxyz')
                angles = np.array([al, be, ga], dtype=np.float32)
                odometries.append(np.concatenate((t, angles)))
        return np.array(images_paths), np.array(odometries, dtype=np.float32)

    def __len__(self):
        return self.size - len(self.sequences)

列车过程代码

train_dataset = VisualOdometryDataLoader(config['datapath'], 192, 640, config['bsize'])
for epoch in range(1, 20):
    for step, (batch_x, batch_y) in enumerate(train_dataset.dataset):
        with tf.GradientTape() as tape:
            with tf.device('/gpu:0'):
                x = flownet(batch_x)
            with tf.device('/cpu:0'):
                t, r = model(x, training=True)
                t_loss_value, r_loss_value = loss_fn(y, t, r)
        grads = tape.gradient([t_loss_value, r_loss_value], model.trainable_variables) # trainable_weights
        optimizer.apply_gradients(zip(grads, model.trainable_variables))