tensorflow:model.evaluate（）和tf.keras.loss.MSE返回完全不同的值

Question

我正在为单变量时间序列预测培训以下RNN

培训后，我想用我的测试集评估模型。我试图用{}和{}用{}得到的预测来实现这一点。然而，结果却完全不同：第一个给出一个十进制值，另一个给出的值要大得多

当使用类tf.keras.losses.MeanSquaredError而不是方法tf.keras.losses.MSE时，会发生类似的情况

数据集是一个tf.data.Dataset对象，y的变量是数组

下面是一个最小的完整可验证示例。代码使用pandas_datareader下载雅虎财务数据。仅保留列Close，并将其转换为tf.data.Dataset，然后对其进行窗口化和移位以创建包含过去值的窗口。使用sklearn.model_selection的train_test_split和一个自定义函数将数据集拆分为train/val/测试集

import tensorflow as tf
import numpy as np
import datetime
from pandas_datareader import data;
from sklearn.model_selection import train_test_split

#get data
everSince = datetime.date(1980, 1, 1)
today = datetime.date(2020, 11, 16)
df = data.get_data_yahoo("CORN", start=everSince,
        end=today);

# extract only 'Close' column
df_close = df[['Close']]

#Convert to TFDataset
WINDOW_SIZE = 10
dataset = tf.data.Dataset.from_tensor_slices((df_close.values))
d = dataset.window(WINDOW_SIZE, shift=1, drop_remainder=True)
d2 = d.flat_map(lambda window: window.batch(WINDOW_SIZE))
len_ds = 0
for item in d2:
    len_ds +=1
d3 = d2.map(lambda window: (window[:-1], window[-1:]))
d_shuffled = d3.shuffle(buffer_size=len_ds, reshuffle_each_iteration=False)

#Split train/val/test
y_targets = np.array([ target.numpy() for _, target in iter(d_shuffled) ])
X_indices = np.arange(len(y_targets))

y_targets = y_targets.reshape((-1,))

#stratify array-like, default=None If not None, data is split in a stratified
#fashion, using this as the class labels.
X_train_indices, X_val_indices, y_train_targets, y_val_targets = train_test_split(
    X_indices, y_targets, test_size=0.15, stratify=None, random_state=53)

X_test_indices, X_val_indices, y_test_targets, y_val_targets = train_test_split(
    X_val_indices, y_val_targets, test_size=0.5, stratify=None, random_state=53)

def get_selected_dataset(ds, X_indices_np):
    # Make a tensor of type tf.int64 to match the one by Dataset.enumerate().
    X_indices_ts = tf.constant(X_indices_np, dtype=tf.int64)

    def is_index_in(index, rest):
        # Returns True if the specified index value is included in X_indices_ts.
        #
        # '==' compares the specified index value with each values in X_indices_ts.
        # The result is a boolean tensor, looks like [ False, True, ..., False ].
        # reduce_any() returns Ture if True is included in the specified tensor.
        return tf.math.reduce_any(index == X_indices_ts)

    def drop_index(index, rest):
        return rest

    # Dataset.enumerate() is similter to Python's enumerate().
    # The method adds indices to each elements. Then, the elements are filtered
    # by using the specified indices. Finally unnecessary indices are dropped.
    selected_ds = ds \
        .enumerate() \
        .filter(is_index_in) \
        .map(drop_index)
    return selected_ds

splitted_train_ds = get_selected_dataset(d_shuffled, X_train_indices)
splitted_val_ds   = get_selected_dataset(d_shuffled, X_val_indices)
splitted_test_ds  = get_selected_dataset(d_shuffled, X_test_indices)


def create_model():
    MODEL_ARCH = [
        tf.keras.layers.GRU(50, return_sequences=True, input_shape=( WINDOW_SIZE-1, 1)),
        tf.keras.layers.GRU(50,),
        tf.keras.layers.Dense(10, activation='tanh'),
        tf.keras.layers.Dense(1, activation='tanh'),
        tf.keras.layers.Lambda(lambda x: x*100)

    ]
    model = tf.keras.models.Sequential(MODEL_ARCH)
    return model


model = create_model()
LR = 1e-3
optimizer = tf.keras.optimizers.Adagrad(lr=LR)
model.compile(loss='mse', optimizer=optimizer) #accuracy is a categorical metric: don't use it

history = model.fit(splitted_train_ds.batch(32), epochs=10,
    validation_data=splitted_val_ds.batch(32), batch_size=32)
model_err = model.evaluate(splitted_test_ds.batch(1))
y_hat = model.predict(splitted_test_ds.batch(1))
y_hat = y_hat.reshape((-1,))
print(model_err )
print(tf.keras.losses.MSE(y_test_targets, y_hat))

我得到的结果是：

0.6533313393592834
237.09732

如果您能就这一重大差异的原因提出任何建议，我们将不胜感激

进一步信息

我认为不同的行为取决于{}。事实上，在下面的代码中使用numpy数组会返回类似的MSE：

import tensorflow as tf
import numpy as np
import datetime
from pandas_datareader import data;
from sklearn.model_selection import train_test_split

#get data
everSince = datetime.date(1980, 1, 1)
today = datetime.date(2020, 11, 16)
df = data.get_data_yahoo("CORN", start=everSince,
        end=today);

# extract only 'Close' column
df_close = df[['Close']]


#Convert to TFDataset
WINDOW_SIZE = 10
x =[]
y = []
vals = df_close.values
for i in range(WINDOW_SIZE, len(vals)):
    y.append(vals[i].flatten())
    x.append(vals[i-WINDOW_SIZE:i].flatten())

x = np.array(x)
y = np.array(y)
idx = np.arange(len(y))
np.random.shuffle(idx)
x_shuffled = x[idx]
y_shuffled = y[idx]
train_size = int(0.7*len(y_shuffled))
val_size = int(0.15*len(y_shuffled))
test_size = int(0.15*len(y_shuffled))

y_train = y_shuffled[:train_size]
x_train = x_shuffled[:train_size]
y_val = y_shuffled[train_size:train_size+val_size]
x_val = x_shuffled[train_size:train_size+val_size]
y_test = y_shuffled[-test_size:]
x_test = x_shuffled[-test_size:]

def create_model(window_size):
    MODEL_ARCH = [
        tf.keras.layers.GRU(50, return_sequences=True, input_shape=( window_size, 1)),
        tf.keras.layers.GRU(50,),
        tf.keras.layers.Dense(10, activation='tanh'),
        tf.keras.layers.Dense(1, activation='tanh'),
        tf.keras.layers.Lambda(lambda x: x*100)

    ]
    model = tf.keras.models.Sequential(MODEL_ARCH)
    return model

def checkOverlappingElements(set1,set2):
    vview = lambda a: np.ascontiguousarray(a).view(np.dtype((np.void, a.dtype.itemsize * a.shape[1])))
    overlapping_els = np.where(vview(set1) == vview(set2).T)
    if overlapping_els[0].size>0:
        print("overlapping elements between x_test a: ", overlapping_els)
        return True
    return False

checkOverlappingElements(x_train, x_val)
checkOverlappingElements(x_train, x_test)
checkOverlappingElements(x_val, x_test)

model = create_model(WINDOW_SIZE)
LR = 1e-3
optimizer = tf.keras.optimizers.Adagrad(lr=LR)
model.compile(loss='mse', optimizer=optimizer)
print(x_train.shape)
x_train = np.expand_dims(x_train, axis=-1)
x_val =   np.expand_dims(x_val, axis=-1)
x_test =   np.expand_dims(x_test, axis=-1)
y_train = np.expand_dims(y_train, axis=-1)
y_val =   np.expand_dims(y_val, axis=-1)
y_test =   np.expand_dims(y_test, axis=-1)
print(x_train.shape, y_train.shape, x_val.shape, y_val.shape, )
#x_test = np.expand_dims(x_test, 0)

history = model.fit(x=x_train, y=y_train, epochs=5, validation_data=(x_val, y_val), batch_size=32)
model_err = model.evaluate(x=x_test, y=y_test, verbose=2, batch_size=1)
y_hat = model.predict(x_test, batch_size=1)
#y_hat = y_hat.reshape((-1,))
print("model.evaluate(): ", model_err)
print("tf.keras.losses.MSE: ", tf.keras.losses.MSE(y_test.reshape(-1, ), y_hat.reshape(-1, )).numpy())

我认为在洗牌数据集时设置reshuffle_each_iteration=False足以避免这些问题，但显然我遗漏了一些东西