我无法使用木偶师和节点为我的 youtube 搜索结果抓取器无限滚动.js

Question

我一直在尝试使用puppeteer和node.js创建一个youtube搜索结果刮板，但我只能刮到结果，即使添加自动滚动功能后，我也无法刮到所有结果。

const puppeteer=require('puppeteer');
async function parseSearch(page){
const search= await page.evaluate(()=>{
const search_results=document.querySelectorAll(".text-wrapper.style-scope.ytd-video-renderer");
const lists=[];
search_results.forEach(elements => {
const vidname=elements.querySelector("h3");
// const quotetext=quoteinfo[0];
const channel=elements.querySelector("a.yt-simple-endpoint.style-scope.yt-formatted-string");
lists.push({
video_name:vidname.innerText,
channel_name:channel.innerHTML
});
});
return lists;
})
console.log(search);
}
async function autoScroll(page)
{
await page.evaluate(async () => {
await new Promise((resolve) => {
var totalHeight = 0;
var distance = 300;
var timer = setInterval(() => {
var scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if(totalHeight >= scrollHeight - window.innerHeight){
clearInterval(timer);
resolve();
}
}, 400);
});
});
}
(async()=>{
const browser=await puppeteer.launch({headless:false});
const page=await browser.newPage();
await page.goto("https://www.youtube.com/results?search_query=programming")

const result=await parseSearch(page);
await autoScroll(page);
console.log(result);
// await browser.close();
})()
console.log("END");

Answer 1

试试这个:

Await page.EvaluateExpressionAsync("window.scrollBy({top: window.innerHeight*1000,behavior: 'smooth'})")