puppeteer iFrame - 类型错误：无法读取未定义的属性"移动"

Question

我正在尝试解决iframe中的captcha。我是否使用以下代码正确访问iframe？我可以点击iframe中的按钮，但不能移动鼠标。我得到了x、y、宽度、高度的位置，但这些位置可能是错误的，因为它在iframe中。如果我获取iframe URL，访问它并解决captcha，那么我就可以移动鼠标并解决capatcha。

感谢

const puppeteer = require('puppeteer')
const fs = require('fs').promises
const Jimp = require('jimp')
const pixelmatch = require('pixelmatch')
const { cv } = require('opencv-wasm')
async function findPuzzlePosition (page) {
let images = await page.$$eval('.geetest_canvas_img canvas', canvases => canvases.map(canvas => canvas.toDataURL().replace(/^data:image/png;base64,/, '')))
await fs.writeFile(`./puzzle.png`, images[1], 'base64')
let srcPuzzleImage = await Jimp.read('./puzzle.png')
let srcPuzzle = cv.matFromImageData(srcPuzzleImage.bitmap)
let dstPuzzle = new cv.Mat()
cv.cvtColor(srcPuzzle, srcPuzzle, cv.COLOR_BGR2GRAY)
cv.threshold(srcPuzzle, dstPuzzle, 127, 255, cv.THRESH_BINARY)
let kernel = cv.Mat.ones(5, 5, cv.CV_8UC1)
let anchor = new cv.Point(-1, -1)
cv.dilate(dstPuzzle, dstPuzzle, kernel, anchor, 1)
cv.erode(dstPuzzle, dstPuzzle, kernel, anchor, 1)
let contours = new cv.MatVector()
let hierarchy = new cv.Mat()
cv.findContours(dstPuzzle, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
let contour = contours.get(0)
let moment = cv.moments(contour)
return [Math.floor(moment.m10 / moment.m00), Math.floor(moment.m01 / moment.m00)]
}
async function findDiffPosition (page) {
await page.waitFor(100)
let srcImage = await Jimp.read('./diff.png')
let src = cv.matFromImageData(srcImage.bitmap)
let dst = new cv.Mat()
let kernel = cv.Mat.ones(5, 5, cv.CV_8UC1)
let anchor = new cv.Point(-1, -1)
cv.threshold(src, dst, 127, 255, cv.THRESH_BINARY)
cv.erode(dst, dst, kernel, anchor, 1)
cv.dilate(dst, dst, kernel, anchor, 1)
cv.erode(dst, dst, kernel, anchor, 1)
cv.dilate(dst, dst, kernel, anchor, 1)
cv.cvtColor(dst, dst, cv.COLOR_BGR2GRAY)
cv.threshold(dst, dst, 150, 255, cv.THRESH_BINARY_INV)
let contours = new cv.MatVector()
let hierarchy = new cv.Mat()
cv.findContours(dst, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
let contour = contours.get(0)
let moment = cv.moments(contour)
return [Math.floor(moment.m10 / moment.m00), Math.floor(moment.m01 / moment.m00)]
}
async function saveSliderCaptchaImages(page) {
await page.waitForSelector('[aria-label="Click to verify"]')
await page.waitFor(3000)
await page.click('[aria-label="Click to verify"]')
await page.waitForSelector('.geetest_canvas_img canvas', { visible: true })
await page.waitFor(1000)
let images = await page.$$eval('.geetest_canvas_img canvas', canvases => {
return canvases.map(canvas => canvas.toDataURL().replace(/^data:image/png;base64,/, ''))
})
await fs.writeFile(`./captcha.png`, images[0], 'base64')
await fs.writeFile(`./original.png`, images[2], 'base64')
}
async function saveDiffImage() {
const originalImage = await Jimp.read('./original.png')
const captchaImage = await Jimp.read('./captcha.png')
const { width, height } = originalImage.bitmap
const diffImage = new Jimp(width, height)
const diffOptions = { includeAA: true, threshold: 0.2 }
pixelmatch(originalImage.bitmap.data, captchaImage.bitmap.data, diffImage.bitmap.data, width, height, diffOptions)
diffImage.write('./diff.png')
}

async function solveCaptcha (page) {
console.log(page)
await saveSliderCaptchaImages(page)
await saveDiffImage()
let [cx, cy] = await findDiffPosition(page)
const sliderHandle = await page.$('.geetest_slider_button')
const handle = await sliderHandle.boundingBox()
await page.waitFor(5000)
console.log(handle)
let xPosition = handle.x + handle.width / 2
let yPosition = handle.y + handle.height / 2
await page.mouse.move(xPosition, yPosition)
await page.mouse.down()
xPosition = handle.x + cx - handle.width / 2
yPosition = handle.y + handle.height / 3
await page.mouse.move(xPosition, yPosition, { steps: 25 })
await page.waitFor(100)
let [cxPuzzle, cyPuzzle] = await findPuzzlePosition(page)
xPosition = xPosition + cx - cxPuzzle
yPosition = handle.y + handle.height / 2
await page.mouse.move(xPosition, yPosition, { steps: 5 })
await page.mouse.up()
await page.waitFor(3000)
// success!
await fs.unlink('./original.png')
await fs.unlink('./captcha.png')
await fs.unlink('./diff.png')
await fs.unlink('./puzzle.png')
}

async function start(){
const browser = await puppeteer.launch({
headless: false,
defaultViewport: { width: 1366, height: 768 },
args: [ '--proxy-server=x.x.x.x:xxx', '--disable-web-security',
'--disable-features=IsolateOrigins,site-per-process'],
})

const page = await browser.newPage()
await page.authenticate({
username: 'xxx',
password: 'xxx',
});

await page.goto('https://someurlwithcaptchainiframe.com', { waitUntil: 'networkidle2' })
await page.waitFor(1000)
await page.content();
try {
innerText = await page.evaluate(() =>  {
return JSON.parse(document.querySelector("body").innerText); 
}); 
}
catch(err) {
console.log('BLOCKED, we want to solve captcha here..')
await page.waitFor(9000)
//frame = await page.mainFrame().childFrames()
//solveCaptcha(frame)
let captchaFrame // this will be populated later by our identified frame
for (const frame of page.mainFrame().childFrames()){
// Here you can use few identifying methods like url(),name(),title()
if (frame.url().includes('geo')){
console.log('we found the captcha iframe')
captchaFrame = frame
console.log('Frame URL: '+captchaFrame._url)
await solveCaptcha(captchaFrame)



// we assign this frame to myFrame to use it later
}
}
console.log('HERE..')
await page.waitFor(90000)


// solveCaptcha(frame)
}
//console.log("innerText now contains the JSON");
//console.log(innerText);


}
start()

Answer 1

您必须使用"。鼠标"；对象来自页面实例，而不是来自iframe，比如这个

if(await page.$('div[id="px-captcha"] > div > iframe[style*="khtml-user-select: none"]') != null){
const frameHandle = await page.$('div[id="px-captcha"] > div > iframe[style*="khtml-user-select: none"]')
const captcha_frame = await frameHandle.contentFrame()   

if(await captcha_frame.$('div[aria-label*="Human Challenge"]') != null){
const example = await captcha_frame.$('div[aria-label*="Human Challenge"]') 
const box = await example.boundingBox()  
await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2) 
await page.mouse.down()
await page.waitForTimeout(5000)
await page.mouse.up()

} 
}