我正在嘗試從https://www.brickeconomy.com/sets/theme/collectable-minifigures抓取資料（只需要每個樂高套裝頁面的網址），但網站上有使用 Javascript __doPostBack 函式的分頁。我查看了許多其他相關答案，知道我需要查看 POST 請求以識別請求表單資料，如下所示：
請求的表單資料的螢屏截圖
我的代碼現在如下：
import requests
from bs4 import BeautifulSoup
url = "http://www.brickeconomy.com/sets/theme/collectable-minifigures"
page_content = requests.get(url).content
soup = BeautifulSoup(page_content, 'html.parser')
VIEWSTATEGENERATOR = soup.find('input',{'id':'__VIEWSTATEGENERATOR'}).get('value')
VIEWSTATE = soup.find('input',{'id':'__VIEWSTATE'}).get('value')
headers = {'user-agent': 'Mozilla/5.0'}
data = {
"ctl00$ScriptManager1": "ctl00$ContentPlaceHolder1$ctlSets$UpdatePanelMain|ctl00$ContentPlaceHolder1$ctlSets$GridViewSets",
"ctl00$txtSearchHeader2": "",
"ctl00$txtSearchHeader": "",
"subthemesorter": "",
"setsorter": "SetNumberDESC",
"ctl00$LoginModalUsername": "",
"ctl00$LoginModalPassword": "",
"__EVENTTARGET": "ctl00$ContentPlaceHolder1$ctlSets$GridViewSets",
"__EVENTARGUMENT": "Page$2",
"__VIEWSTATE":VIEWSTATE,
"__VIEWSTATEGENERATOR": VIEWSTATEGENERATOR,
"__ASYNCPOST": 'true'
}
res = requests.post(url, data=data, headers =headers).content
BeautifulSoup(res, 'html.parser').find_all(class_ = 'mb-5')
但是，它仍然顯示第一頁的資料。非常感謝這里的任何建議。謝謝！
uj5u.com熱心網友回復：
您將發布請求發送到錯誤的網址。一旦我用正確的網址替換了您現有的網址，腳本就開始作業了：
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
base = 'https://www.brickeconomy.com'
start_url = 'http://www.brickeconomy.com/sets/theme/collectable-minifigures'
post_url = 'https://www.brickeconomy.com/sets/theme/sets/theme/collectable-minifigures'
data = {
"ctl00$ScriptManager1": "ctl00$ContentPlaceHolder1$ctlSets$UpdatePanelMain|ctl00$ContentPlaceHolder1$ctlSets$GridViewSets",
"ctl00$txtSearchHeader2": "",
"ctl00$txtSearchHeader": "",
"subthemesorter": "",
"setsorter": "SetNumberDESC",
"ctl00$LoginModalUsername": "",
"ctl00$LoginModalPassword": "",
"__EVENTTARGET": "ctl00$ContentPlaceHolder1$ctlSets$GridViewSets",
"__EVENTARGUMENT": "Page$2",
"__ASYNCPOST": 'true'
}
with requests.Session() as s:
s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'
r = s.get(start_url)
soup = BeautifulSoup(r.text,"lxml")
data['__VIEWSTATE'] = soup.find('input',{'id':'__VIEWSTATE'}).get('value')
data['__VIEWSTATEGENERATOR'] = soup.find('input',{'id':'__VIEWSTATEGENERATOR'}).get('value')
res = s.post(post_url,data=data)
soup = BeautifulSoup(res.text,"lxml")
for item in soup.select("table.table > tr h4 > a"):
inner_url = urljoin(base,item.get("href"))
print(inner_url)
轉載請註明出處，本文鏈接：https://www.uj5u.com/yidong/344494.html