T5Tokenizer和T5EncoderModel用于对句子进行编码,然后对nn进行编码.TransformerDec



使用T5Tokenizer标记句子,然后使用T5EncoderModel进行编码。最后,使用pytorch nn。TransformerDecoder对其进行解码。目标向量是torch.ttensor[y1,y2],其中y1和y2具有二进制值。但它返回一个错误RuntimeError: DataLoader worker (pid(s) 723820) exited unexpectedly。数据集类如下所示。

tokenizer=T5TokenizerFast.from_pretrained('t5-base')
MAX_LEN = tokenizer(balanced_sentences[max_idx]).input_ids
class MyDataset(Dataset):
def __init__(self, sentences, labels):
self.labels = labels
self.sentences = sentences
self.tokenizer = tokenizer
def __len__(self):
return len(self.labels)

def __getitem__(self, idx):
sentence = self.sentences[idx]
label = self.labels[idx]
tokenized = tokenizer(self.sentences[idx],
max_length=MAX_LEN,
padding='max_length', 
truncation=True,
add_special_tokens=True,
return_tensors="pt")
sample = {"input_id": tokenized['input_ids'].flatten(), 
"attention_mask":tokenized['attention_mask'].flatten(), 
"label": label}
return sample

模型以以下方式创建

class MyModel(pl.LightningModule):
def __init__(self):
super().__init__()
self.encoder = T5EncoderModel.from_pretrained("t5-base", return_dict = True)
dec_layer = nn.TransformerDecoderLayer(d_model=512, nhead=8, batch_first=True)
self.decoder = nn.TransformerDecoder(dec_layer, num_layers=6)
self.loss_fc = nn.BCELoss()

def forward(self, input_id, attention_mask, label=None):
encoded_sentence = self.encoder(input_id, attention_mask)#.last_hidden_state.mean(dim=1) 
loss, output = self.decoder(label, encoded_sentence)
return loss, output

def training_step(self, batch, batch_idx):
input_id = batch['input_id']
attention_mask = batch['attention_mask']
label = batch['label']    
loss, output = self(input_id = input_id,
attention_mask=attention_mask,
label = label)
self.log('train_loss', loss, prog_bar = True, logger = True)
return loss

def validation_step(self, batch, batch_idx):
input_id = batch['input_id']
attention_mask = batch['attention_mask']        
label = batch['label']    
loss, output = self(input_id = input_id,
attention_mask=attention_mask,
label = label)
self.log('train_loss', loss, prog_bar = True, logger = True)
return loss

def configure_optimizers(self):
return torch.optim.Adam(self.parameters(), lr=0.001)
trainer = pl.Trainer(logger = logger, 
num_sanity_val_steps=0, 
max_epochs = N_EPOCHS, 
gpus = 1, 
progress_bar_refresh_rate = 20)
trainer.fit(model, data_module)

如果您使用VSCode,听起来数据加载程序有多个工作线程,无法创建更多线程。将工作人员数量设置为0。

请参阅:PyTorch DataLoader的VSCode错误?

相关内容

  • 没有找到相关文章

最新更新