在超参数优化Python中找不到泄漏的ReLU

Question

我在神经网络中使用超参数字典和超参数优化函数，如下所示：

from tensorflow.keras.layers import LeakyReLU
parameters=[
{
"name": "learning_rate",
"type": "range",
"bounds": [0.001, 0.5],
"log_scale": True,
},
{
"name": "dropout_rate",
"type": "range",
"bounds": [0.01, 0.9],
"log_scale": True,
},
{
"name": "num_hidden_layers",
"type": "range",
"bounds": [1, 7],
"value_type": "int"
},
{
"name": "neurons_per_layer",
"type": "range",
"bounds": [1, 300],
"value_type": "int"
},
{
"name": "batch_size",
"type": "choice",
"values": [8, 10, 16, 20, 30],
},

{
"name": "activation",
"type": "choice",
"values": [ 'LeakyReLU(alpha=0.3)', 'relu'],
},
{
"name": "optimizer",
"type": "choice",
"values": ['adam', 'rms', 'sgd'],
},
]
# This returns a multi-layer-perceptron model in Keras.
def get_keras_model(num_hidden_layers, 
num_neurons_per_layer, 
dropout_rate, 
activation):
# create the MLP model.

# define the layers.
inputs = tf.keras.Input(shape=(train_dataset.shape[1],))  # input layer.
x = layers.Dropout(dropout_rate)(inputs) # dropout on the weights.

# Add the hidden layers.
for i in range(num_hidden_layers):
x = layers.Dense(num_neurons_per_layer, 
activation=activation)(x)
x = layers.Dropout(dropout_rate)(x)

# output layer.
outputs = layers.Dense(1, activation='linear')(x)

model = tf.keras.Model(inputs=inputs, outputs=outputs)
return model

# This function takes in the hyperparameters and returns a score (Cross validation). 
# Returns the mean of the validation loss based on which we decide which algorithm has the best hyperparameters
def keras_mlp_cv_score(parameterization, weight=None):

model = get_keras_model(parameterization.get('num_hidden_layers'),
parameterization.get('neurons_per_layer'),
parameterization.get('dropout_rate'),
parameterization.get('activation'))

opt = parameterization.get('optimizer')
opt = opt.lower()

learning_rate = parameterization.get('learning_rate')

if opt == 'adam':
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
elif opt == 'rms':
optimizer = tf.keras.optimizers.RMSprop(learning_rate=learning_rate)
else:
optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)

act = parameterization.get('activation')
act = act.lower()

if act == 'leakyrelu': 
activation = ""
get_keras_model.add(tf.layers.leakyReLU())


NUM_EPOCHS = 100

# Specify the training configuration.
model.compile(optimizer=optimizer,
loss=tf.keras.losses.MeanSquaredError(),
metrics=['mae', 'mse'] )
data = X_train
labels = y_train.values

early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)

# fit the model using a 20% validation set. with a patience of 10 to avoid overfitting
res = model.fit(data, labels, epochs=NUM_EPOCHS, batch_size=parameterization.get('batch_size'),
validation_split=0.2, callbacks=[early_stop, tfdocs.modeling.EpochDots()])

# look at the last 10 epochs. Get the mean and standard deviation of the validation score.
last10_scores = np.array(res.history['val_loss'][-10:])
mean = last10_scores.mean()
sem = last10_scores.std()

# If the model didn't converge then set a high loss.
if np.isnan(mean):
return 9999.0, 0.0

return mean, sem

但无论我如何使用LeakyReLU，它都会抛出未找到激活函数的错误。我也试过tf.nn.leaky_relu请帮助我将LeakyReLU正确地纳入我的代码中。

Answer 1

您写道：

act = act.lower()    
if act == 'LeakyReLU': 
...

这个测试总是错误的，因为'LeakyReLU'有一些大写字母，而act从来没有，因此它永远不会在模型中添加LeakyRelu层。

尝试：

act = act.lower()    
if act == 'leakyrelu': 
...

此外，正确的语法是tf.keras.layers.LeakyReLU()(tensorflow文档中的Leaky ReLU(。

如果需要，请更新您的问题与堆栈的错误。

请尝试这个：

def get_keras_model(num_hidden_layers, 
num_neurons_per_layer, 
dropout_rate, 
activation):
# create the MLP model.
# define the layers.
inputs = tf.keras.Input(shape=(train_dataset.shape[1],))  # input layer.
x = layers.Dropout(dropout_rate)(inputs) # dropout on the weights.
leakyrelu =  activation.lower() == 'leakyrelu'
# Add the hidden layers.
for i in range(num_hidden_layers):
x = layers.Dense(num_neurons_per_layer, 
activation=activation if not leakyrelu else None)(x)
if leakyrelu:
x = LeakyReLU()(x)
x = layers.Dropout(dropout_rate)(x)    

def keras_mlp_cv_score(parameterization, weight=None):
model = get_keras_model(parameterization.get('num_hidden_layers'),
parameterization.get('neurons_per_layer'),
parameterization.get('dropout_rate'),
parameterization.get('activation'))
# No need for "if act == 'leakyrelu':" etc anymore.
....