使用T5Tokenizer标记句子,然后使用T5EncoderModel进行编码。最后,使用pytorch nn。TransformerDecoder对其进行解码。目标向量是torch.ttensor[y1,y2],其中y1和y2具有二进制值。但它返回一个错误RuntimeError: DataLoader worker (pid(s) 723820) exited unexpectedly
。数据集类如下所示。
tokenizer=T5TokenizerFast.from_pretrained('t5-base')
MAX_LEN = tokenizer(balanced_sentences[max_idx]).input_ids
class MyDataset(Dataset):
def __init__(self, sentences, labels):
self.labels = labels
self.sentences = sentences
self.tokenizer = tokenizer
def __len__(self):
return len(self.labels)
def __getitem__(self, idx):
sentence = self.sentences[idx]
label = self.labels[idx]
tokenized = tokenizer(self.sentences[idx],
max_length=MAX_LEN,
padding='max_length',
truncation=True,
add_special_tokens=True,
return_tensors="pt")
sample = {"input_id": tokenized['input_ids'].flatten(),
"attention_mask":tokenized['attention_mask'].flatten(),
"label": label}
return sample
模型以以下方式创建
class MyModel(pl.LightningModule):
def __init__(self):
super().__init__()
self.encoder = T5EncoderModel.from_pretrained("t5-base", return_dict = True)
dec_layer = nn.TransformerDecoderLayer(d_model=512, nhead=8, batch_first=True)
self.decoder = nn.TransformerDecoder(dec_layer, num_layers=6)
self.loss_fc = nn.BCELoss()
def forward(self, input_id, attention_mask, label=None):
encoded_sentence = self.encoder(input_id, attention_mask)#.last_hidden_state.mean(dim=1)
loss, output = self.decoder(label, encoded_sentence)
return loss, output
def training_step(self, batch, batch_idx):
input_id = batch['input_id']
attention_mask = batch['attention_mask']
label = batch['label']
loss, output = self(input_id = input_id,
attention_mask=attention_mask,
label = label)
self.log('train_loss', loss, prog_bar = True, logger = True)
return loss
def validation_step(self, batch, batch_idx):
input_id = batch['input_id']
attention_mask = batch['attention_mask']
label = batch['label']
loss, output = self(input_id = input_id,
attention_mask=attention_mask,
label = label)
self.log('train_loss', loss, prog_bar = True, logger = True)
return loss
def configure_optimizers(self):
return torch.optim.Adam(self.parameters(), lr=0.001)
trainer = pl.Trainer(logger = logger,
num_sanity_val_steps=0,
max_epochs = N_EPOCHS,
gpus = 1,
progress_bar_refresh_rate = 20)
trainer.fit(model, data_module)
如果您使用VSCode,听起来数据加载程序有多个工作线程,无法创建更多线程。将工作人员数量设置为0。
请参阅:PyTorch DataLoader的VSCode错误?