因此，我一直在尝试从头开始构建和训练一个用于移情对话生成任务的Transformer模型，目前我在训练过程中遇到了困难，因为该模型似乎只能预测最终输出层中的START和END令牌，而与提供给Transformer解码器的目标令牌无关。我已经完成了多次实现，发现并纠正了一些问题(主要是多头注意层和标记化(，但仍然没有任何运气。
我使用F.cross_entropy来计算从变换器out[:, :-1:, :]输出的最终logits和我的数据集中的目标序列target[:, 1:]之间的交叉熵。移位当然是必要的，因为变换器的每个输出对应于下一个预测的令牌。我尝试删除这个丢失函数(即out[:, :-2:, :] and target[:, 1:-1](中的START和END标记，但这也没有帮助。logits和target都是根据PyTorch文档成形的，即分别为(batch_size，classes，sequence_length(和(batch_size，sequence_length(，其中target包含类索引(因此忽略填充索引(。训练输出看起来像这样。
Epoch 0: 1%| | 1/180 [00:05<16:28, 5.53s/it, loss=11, v_num=3, train_loss=11.00]
Epoch 0: 1%| | 2/180 [00:25<37:55, 12.78s/it, loss=11, v_num=3, train_loss=11.00]
...
Epoch 5: 90%|█████████ | 162/180 [00:58<00:06, 2.77it/s, loss=5.54, v_num=3, train_loss=5.520]
Epoch 5: 90%|█████████ | 162/180 [00:58<00:06, 2.77it/s, loss=5.53, v_num=3, train_loss=5.430]
如上所述，损耗衰减到5-6之间的恒定损耗值，并且保持恒定(甚至直到第50个历元(。我通过软最大化logits打印出每个训练步骤的概率张量。最高概率归因于START和END令牌，而与进入变换器解码器的目标令牌无关。
为了证实这种行为，我编写了一个脚本，在给定上下文序列的情况下(使用波束搜索(预测训练模型的响应，并将第一个目标令牌设置为[START]。无论我向模型输入什么上下文序列或使用什么波束宽度，下一个目标标记总是被预测为[END]。我不确定这是否与标记化或模型中的一些权重爆炸有关，但我似乎无法摆脱这种行为。我甚至加入了脱落层来消除后一个问题，但仍然没有运气。即使我删除了在解码器中添加的情感嵌入，这个问题仍然存在。
以下是模型的完整实现供参考：
class MultiHeadAttention(nn.Module):
def __init__(self, embed_size: int, heads: int) -> None:
super().__init__()
self.embed_size = embed_size
self.heads = heads
self.head_dim = self.embed_size // self.heads
assert self.head_dim * self.heads == self.embed_size
self.values = nn.Linear(self.embed_size, self.embed_size, bias=False)
self.keys = nn.Linear(self.embed_size, self.embed_size, bias=False)
self.queries = nn.Linear(self.embed_size, self.embed_size, bias=False)
self.fc_out = nn.Linear(self.embed_size, self.embed_size, bias=False)
def forward(
self,
keys: torch.Tensor,
values: torch.Tensor,
queries: torch.Tensor,
mask: torch.Tensor
) -> torch.Tensor:
N = queries.shape[0]
keys_len, values_len, queries_len = keys.shape[1], values.shape[1], queries.shape[1]
values = self.values(values).reshape(N, values_len, self.heads, self.head_dim)
keys = self.keys(keys).reshape(N, keys_len, self.heads, self.head_dim)
queries = self.queries(queries).reshape(N, queries_len, self.heads, self.head_dim)
scores = torch.einsum("nqhd,nkhd->nhqk", [queries, keys])
# Apply mask to attention scores if specified
if mask is not None:
scores = scores.masked_fill(mask == 0, float("-1e20"))
# Normalise with respect to all keys
attention = F.softmax(scores / (self.embed_size ** 0.5), dim=-1)
out = torch.einsum("nhqk,nvhd->nqhd", [attention, values])
out = self.fc_out(out.reshape(N, queries_len, self.embed_size))
return out
class TransformerBlock(nn.Module):
def __init__(
self,
embed_size: int,
heads: int,
dropout: float,
forward_expansion: int
) -> None:
super().__init__()
self.attention = MultiHeadAttention(embed_size, heads)
self.norm1 = nn.LayerNorm(embed_size)
self.norm2 = nn.LayerNorm(embed_size)
self.dropout = nn.Dropout(dropout)
self.ff = nn.Sequential(
nn.Linear(embed_size, embed_size * forward_expansion),
nn.ReLU(),
nn.Linear(embed_size * forward_expansion, embed_size)
)
def forward(
self,
keys: torch.Tensor,
values: torch.Tensor,
queries: torch.Tensor,
mask: torch.Tensor
) -> torch.Tensor:
attention = self.attention(keys, values, queries, mask)
contextualised = self.dropout(self.norm1(attention + queries))
forward = self.ff(contextualised)
out = self.dropout(self.norm2(forward + contextualised))
return out
class Encoder(nn.Module):
def __init__(
self,
vocab_size: int,
padding_idx: int,
num_layers: int,
embed_size: int,
heads: int,
dropout: float,
forward_expansion: int,
max_seq_len: int,
num_of_emo_labels: int
) -> None:
super().__init__()
self.word_embeddings = nn.Embedding(
vocab_size + 1, embed_size, padding_idx=padding_idx)
self.pos_embeddings = nn.Embedding(max_seq_len, embed_size)
self.ds_embeddings = nn.Embedding(2 + 1, embed_size, padding_idx=0)
self.layers = nn.ModuleList(
[TransformerBlock(embed_size, heads, dropout, forward_expansion)
for _ in range(num_layers)]
)
self.dropout = nn.Dropout(dropout)
def forward(
self,
context: torch.Tensor,
context_ds_state: torch.Tensor,
mask: torch.Tensor,
emotion_label: torch.Tensor
) -> torch.Tensor:
N, seq_len = context.shape
positions = torch.arange(0, seq_len, device=context.device).expand(N, seq_len)
word_embeddings = self.word_embeddings(context)
pos_embeddings = self.pos_embeddings(positions)
ds_embeddings = self.ds_embeddings(context_ds_state)
out = self.dropout(word_embeddings + pos_embeddings + ds_embeddings)
for layer in self.layers:
out = layer(out, out, out, mask)
return out
class DecoderBlock(nn.Module):
def __init__(
self,
embed_size: int,
heads: int,
dropout: float,
forward_expansion: int
) -> None:
super().__init__()
self.attention = MultiHeadAttention(embed_size, heads)
self.norm = nn.LayerNorm(embed_size)
self.transformer_block = TransformerBlock(
embed_size,
heads,
dropout,
forward_expansion
)
self.dropout = nn.Dropout(dropout)
def forward(
self,
x: torch.Tensor,
keys: torch.Tensor,
values: torch.Tensor,
target_mask: torch.Tensor,
input_mask: torch.Tensor
) -> torch.Tensor:
attention = self.attention(x, x, x, target_mask)
queries = self.dropout(self.norm(attention + x))
out = self.transformer_block(keys, values, queries, input_mask)
return out
class Decoder(nn.Module):
def __init__(
self,
vocab_size: int,
padding_idx: int,
num_layers: int,
embed_size: int,
heads: int,
dropout: float,
forward_expansion: int,
max_seq_len: int,
num_of_emo_labels: int
) -> None:
super().__init__()
self.word_embeddings = nn.Embedding(
vocab_size + 1, embed_size, padding_idx=padding_idx)
self.pos_embeddings = nn.Embedding(max_seq_len, embed_size)
self.ds_embeddings = nn.Embedding(2 + 1, embed_size, padding_idx=0)
self.emotion_embedding = nn.Embedding(num_of_emo_labels, embed_size)
self.layers = nn.ModuleList(
[DecoderBlock(embed_size, heads, dropout, forward_expansion)
for _ in range(num_layers)]
)
self.dropout = nn.Dropout(dropout)
self.fc_out = nn.Linear(embed_size, vocab_size)
def forward(
self,
target: torch.Tensor,
target_ds_state: torch.Tensor,
encoder_out: torch.Tensor,
target_mask: torch.Tensor,
input_mask: torch.Tensor,
emotion_label: torch.Tensor
) -> torch.Tensor:
N, seq_len = target.shape
positions = torch.arange(0, seq_len, device=target.device).expand(N, seq_len)
word_embeddings = self.word_embeddings(target)
pos_embeddings = self.pos_embeddings(positions)
ds_embeddings = self.ds_embeddings(target_ds_state)
out = self.dropout(word_embeddings + pos_embeddings + ds_embeddings)
for layer in self.layers:
out = layer(out, encoder_out, encoder_out, target_mask, input_mask)
emotion_embedding = self.emotion_embedding(
emotion_label).unsqueeze(1).expand(-1, seq_len, -1)
out = self.fc_out(out + emotion_embedding)
return out
class Transformer(nn.Module):
def __init__(
self,
vocab_size: int,
num_of_emo_labels: int,
max_seq_len: int,
padding_idx: int,
num_layers: int = 6,
embed_size: int = 256,
heads: int = 8,
dropout: float = 0.5,
forward_expansion: int = 4
) -> None:
super().__init__()
self.padding_idx = padding_idx
self.encoder = Encoder(
vocab_size,
padding_idx,
num_layers,
embed_size,
heads,
dropout,
forward_expansion,
max_seq_len,
num_of_emo_labels
)
self.decoder = Decoder(
vocab_size,
padding_idx,
num_layers,
embed_size,
heads,
dropout,
forward_expansion,
max_seq_len,
num_of_emo_labels
)
def create_padding_mask(self, batch_seq):
N = batch_seq.size(dim=0)
padding_mask = (batch_seq != self.padding_idx).unsqueeze(1).unsqueeze(2)
return padding_mask
def create_lookahead_mask(self, batch_seq):
N, seq_len = batch_seq.shape
lookahead_mask = torch.tril(torch.ones(
N, 1, seq_len, seq_len, device=batch_seq.device))
return lookahead_mask
def forward(
self,
context: torch.Tensor,
target: torch.Tensor,
context_ds_state: torch.Tensor,
target_ds_state: torch.Tensor,
emotion_label: torch.Tensor
) -> None:
input_mask = self.create_padding_mask(context)
target_mask = torch.minimum(
self.create_lookahead_mask(target),
self.create_padding_mask(target)
)
encoder_out = self.encoder(
context,
context_ds_state,
input_mask,
emotion_label
)
out = self.decoder(
target,
target_ds_state,
encoder_out,
target_mask,
input_mask,
emotion_label
)
return out
如果相关的话，我已经使用Adam和AdamW作为StepLR调度器的优化器。我已经被这个问题困扰了一段时间，所以如果有任何帮助，我将不胜感激。提前感谢：(
我在使用nn时也遇到了类似的问题。变压器模块。在对训练方法进行了大量更改后，我决定大幅降低的初始学习率