python html抓取JS函数的结果

Question

我正在尝试抓取一个执行JS脚本的网页。

我可以使用from requests import get获得不执行JS的HTML

但我不能像用mozilla 检查网页时那样得到JS函数的结果

这里是我想从中获得结果的函数：

function showFlight(idPilote, idFlight, idActivite)

在这个页面上

知道怎么做吗？

我试过这个

def kmlScrape(target):
curl = pycurl.Curl()
curl.setopt(pycurl.CAINFO, certifi.where())
curl.setopt(pycurl.SSL_VERIFYPEER, 0) 
curl.setopt(pycurl.URL, target)
curl.setopt(pycurl.WRITEFUNCTION, e.write)
curl.perform()
return e

或

response = get(target)
print(response.text)
html_soup = BeautifulSoup(response.text, 'html.parser')

我想做的是用pycurl格式编写这个curl命令：

curl "https://www.syride.com/scripts/ajx_vols.php?p=/en/flights/&idPays=0&pseudo=0&typePratique=0&page=01&idSpot=0&recherche=&order=&tri=&l=en" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0" -H "Accept: */*" -H "Accept-Language: en-US,en;q=0.5" --compressed -H "X-Requested-With: XMLHttpRequest" -H "DNT: 1" -H "Connection: keep-alive" -H "Referer: https://www.syride.com/en/flights/&idPays=0&pseudo=0&typePratique=0&page=01&idSpot=0&recherche=&order=&tri=" -H "Cookie: instruments2=5ai6jgg9p6bpkh3m7slf0b2vq7; gb__ssyride=aufnlupslr259o49nc98qm1t55" -H "Cache-Control: max-age=0"

任何帮助都非常欢迎。

谢谢，Mat

Answer 1

这是对Back-EndAPI的直接调用。

import requests
import pandas as pd

params = {
'l': 'en'
}

def main(url, params):
with requests.Session() as req:
allin = []
for page in range(1, 41):
print(F"Extracting Page# {page}")
params['page'] = page
r = req.get(url, params=params)
df = pd.read_html(r.content, encoding="UTF-8",
header=0)[0][:-1]
allin.append(df)
new = pd.concat(allin)
new.to_csv("data.csv", index=False)
print("Done")

main("https://www.syride.com/scripts/ajx_vols.php", params)

Answer 2

我找到了答案：

def kmlScrape(target):
curl = pycurl.Curl()
curl.setopt(pycurl.CAINFO, certifi.where())
curl.setopt(pycurl.SSL_VERIFYPEER, 0) 
curl.setopt(pycurl.URL, target)
opts={pycurl.USERAGENT:"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0", pycurl.HTTPHEADER:["Accept: */*", "Accept-Language: en-US,en;q=0.5", "Connection: keep-alive", "X-Requested-With: XMLHttpRequest", "DNT: 1", "Referer: https://www.syride.com/en/flights/&idPays=0&pseudo=0&typePratique=0&page=01&idSpot=0&recherche=&order=&tri=", "Cookie: instruments2=5ai6jgg9p6bpkh3m7slf0b2vq7; gb__ssyride=aufnlupslr259o49nc98qm1t55", "Cache-Control: max-age=0"]}
for (key,value) in opts.items():
curl.setopt(key,value)
curl.setopt(pycurl.WRITEFUNCTION, e.write)
curl.perform()
return e