Optuna Hyperband算法未按预期模型训练方案执行

Question

我在使用Hyperband算法和Optuna时发现了一个问题。根据Hyperband算法，当min_resources设置为5，max_resources设置为20，reduction_factor设置为2时，搜索应该从1号组的4个模型开始，每个模型在第一轮中训练5个周期。接下来，每轮模型的数量会减少一半，也就是每个后续组的搜索空间也会减少一半。例如，2号组的初始搜索空间应该是2个模型，而剩下的模型在每轮中的训练周期会翻倍。因此，预计总模型数量应该是11个，但实际上却训练了很多模型。

文章链接：- https://arxiv.org/pdf/1603.06560.pdf

import optuna
import numpy as np
import pandas as pd 
from tensorflow.keras.layers import Dense,Flatten,Dropout
import tensorflow as tf
from tensorflow.keras.models import Sequential


# Toy dataset generation
def generate_toy_dataset():
    np.random.seed(0)
    X_train = np.random.rand(100, 10)
    y_train = np.random.randint(0, 2, size=(100,))
    X_val = np.random.rand(20, 10)
    y_val = np.random.randint(0, 2, size=(20,))
    return X_train, y_train, X_val, y_val

X_train, y_train, X_val, y_val = generate_toy_dataset()

# Model building function
def build_model(trial):
    model = Sequential()
    model.add(Dense(units=trial.suggest_int('unit_input', 20, 30),
                    activation='selu',
                    input_shape=(X_train.shape[1],)))

    num_layers = trial.suggest_int('num_layers', 2, 3)
    for i in range(num_layers):
        units = trial.suggest_int(f'num_layer_{i}', 20, 30)
        activation = trial.suggest_categorical(f'activation_layer_{i}', ['relu', 'selu', 'tanh'])
        model.add(Dense(units=units, activation=activation))
        if trial.suggest_categorical(f'dropout_layer_{i}', [True, False]):
            model.add(Dropout(rate=0.5))

    model.add(Dense(1, activation='sigmoid'))

    optimizer_name = trial.suggest_categorical('optimizer', ['adam', 'rmsprop'])
    if optimizer_name == 'adam':
        optimizer = tf.keras.optimizers.Adam()
    else:
        optimizer = tf.keras.optimizers.RMSprop()

    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy', tf.keras.metrics.AUC(name='val_auc')])

    return model

def objective(trial):
    model = build_model(trial)
    # Assuming you have your data prepared
    # Modify the fit method to include AUC metric
    history = model.fit(X_train, y_train, validation_data=(X_val, y_val), verbose=1)
    
    # Check if 'val_auc' is recorded
    auc_key = None
    for key in history.history.keys():
        if key.startswith('val_auc'):
            auc_key = key
            print(f"auc_key is {auc_key}")
            break
    
    if auc_key is None:
        raise ValueError("AUC metric not found in history. Make sure it's being recorded during training.")
    
    # Report validation AUC for each model
    
    if auc_key =="val_auc":
        step=0
    else:
        step = int(auc_key.split('_')[-1])
    
    auc_value=history.history[auc_key][0]
    trial.report(auc_value, step=step)
    print(f"prune or not:-{trial.should_prune()}")
    if trial.should_prune():
        raise optuna.TrialPruned()

    return history.history[auc_key]

# Optuna study creation
study = optuna.create_study(
    direction='maximize',
    pruner=optuna.pruners.HyperbandPruner(
        min_resource=5,
        max_resource=20,
        reduction_factor=2
    )
)

# Start optimization
study.optimize(objective)