如何使用TensorFlow tf.train.string_input_producer生成多个时代的数据？

Question

当我想使用tf.train.string_input_producer加载两个阶段的数据时，我使用

filename_queue = tf.train.string_input_producer(filenames=['data.csv'], num_epochs=2, shuffle=True)

col1_batch, col2_batch, col3_batch = tf.train.shuffle_batch([col1, col2, col3], batch_size=batch_size, capacity=capacity,\min_after_dequeue=min_after_dequeue, allow_smaller_final_batch=True)

但后来我发现这次行动并没有产生我想要的东西。

它只能在data.csv中生成2次每个样本，但生成的顺序不清楚。例如，data.csv中的3行数据

[[1]
[2]
[3]]

它将产生（每个样本只出现2次，但顺序是可选的）

[1]
[1]
[3]
[2]
[2]
[3]

但我想要的是（每个时代是分开的，在每个时代洗牌）

(epoch 1:)
[1]
[2]
[3]
(epoch 2:)
[1]
[3]
[2]

另外，如何知道一个纪元是什么时候完成的？有标志变量吗？谢谢！

我的密码在这里。

import tensorflow as tf

def read_my_file_format(filename_queue):
    reader = tf.TextLineReader()
    key, value = reader.read(filename_queue)
    record_defaults = [['1'], ['1'], ['1']]  
    col1, col2, col3 = tf.decode_csv(value, record_defaults=record_defaults, field_delim='-')
    # col1 = list(map(int, col1.split(',')))
    # col2 = list(map(int, col2.split(',')))
    return col1, col2, col3

def input_pipeline(filenames, batch_size, num_epochs=1):
  filename_queue = tf.train.string_input_producer(
    filenames, num_epochs=num_epochs, shuffle=True)
  col1,col2,col3 = read_my_file_format(filename_queue)

  min_after_dequeue = 10
  capacity = min_after_dequeue + 3 * batch_size
  col1_batch, col2_batch, col3_batch = tf.train.shuffle_batch(
    [col1, col2, col3], batch_size=batch_size, capacity=capacity,
    min_after_dequeue=min_after_dequeue, allow_smaller_final_batch=True)
  return col1_batch, col2_batch, col3_batch

filenames=['1.txt']
batch_size = 3
num_epochs = 1
a1,a2,a3=input_pipeline(filenames, batch_size, num_epochs)

with tf.Session() as sess:
  sess.run(tf.local_variables_initializer())
  # start populating filename queue
  coord = tf.train.Coordinator()
  threads = tf.train.start_queue_runners(coord=coord)
  try:
    while not coord.should_stop():
      a, b, c = sess.run([a1, a2, a3])
      print(a, b, c)
  except tf.errors.OutOfRangeError:
    print('Done training, epoch reached')
  finally:
    coord.request_stop()

  coord.join(threads) 

我的数据是

1,2-3,4-A
7,8-9,10-B
12,13-14,15-C
17,18-19,20-D
22,23-24,25-E
27,28-29,30-F
32,33-34,35-G
37,38-39,40-H

Answer 1

你可能想看看这个answer类似的问题。

简而言之：

• 如果num_epochs>；1，则所有数据同时排队，并且不受纪元的影响，

• 所以你没有能力监视哪个时代正在退出队列。

您可以做的是引用的答案中的第一个建议，即使用num_epochs==1，并在每次运行时重新初始化本地队列变量（显然不是模型变量）。

init_queue = tf.variables_initializer(tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope='input_producer'))
with tf.Session() as sess: 
    sess.run(tf.global_variables_initializer())
    sess.run(tf.local_variables_initializer())
for e in range(num_epochs):
    with tf.Session() as sess:
       sess.run(init_queue) # reinitialize the local variables in the input_producer scope
       # start populating filename queue
       coord = tf.train.Coordinator()
       threads = tf.train.start_queue_runners(coord=coord)
       try:
           while not coord.should_stop():
               a, b, c = sess.run([a1, a2, a3])
               print(a, b, c)
       except tf.errors.OutOfRangeError:
           print('Done training, epoch reached')
       finally:
           coord.request_stop()

       coord.join(threads)

Answer 2

作为Nicolas observes，^{}API不提供检测何时到达一个epoch结尾的能力，而是将所有epoch连接到一个长批中。出于这个原因，我们最近（在TensorFlow 1.2中）添加了^{} API，这使得表达更复杂的管道成为可能，包括您的用例。

下面的代码片段显示了如何使用tf.contrib.data编写程序：

import tensorflow as tf

def input_pipeline(filenames, batch_size):
    # Define a `tf.contrib.data.Dataset` for iterating over one epoch of the data.
    dataset = (tf.contrib.data.TextLineDataset(filenames)
               .map(lambda line: tf.decode_csv(
                    line, record_defaults=[['1'], ['1'], ['1']], field_delim='-'))
               .shuffle(buffer_size=10)  # Equivalent to min_after_dequeue=10.
               .batch(batch_size))

    # Return an *initializable* iterator over the dataset, which will allow us to
    # re-initialize it at the beginning of each epoch.
    return dataset.make_initializable_iterator() 

filenames=['1.txt']
batch_size = 3
num_epochs = 10
iterator = input_pipeline(filenames, batch_size)

# `a1`, `a2`, and `a3` represent the next element to be retrieved from the iterator.    
a1, a2, a3 = iterator.get_next()

with tf.Session() as sess:
    for _ in range(num_epochs):
        # Resets the iterator at the beginning of an epoch.
        sess.run(iterator.initializer)

        try:
            while True:
                a, b, c = sess.run([a1, a2, a3])
                print(a, b, c)
        except tf.errors.OutOfRangeError:
            # This will be raised when you reach the end of an epoch (i.e. the
            # iterator has no more elements).
            pass                 

        # Perform any end-of-epoch computation here.
        print('Done training, epoch reached')