我試圖刮掉沃爾瑪，如果我只是以requests格式轉儲 curl Payload ，它作業正常，問題是cookies，如果我省略，它會給出 403 錯誤。我不想傳遞靜態 cookie，我試圖傳遞會話 cookie 但沒有作業。下面是我的代碼
傳遞靜態 Cookie
import requests
headers = {
'authority': 'www.walmart.com',
'pragma': 'no-cache',
'cache-control': 'no-cache',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"',
'dnt': '1',
'sec-ch-ua-mobile': '?0',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36',
'sec-ch-ua-platform': '"macOS"',
'content-type': 'application/json',
'accept': '*/*',
'sec-fetch-site': 'same-origin',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://www.walmart.com/store/5939-bellevue-wa/search?query=butter',
'accept-language': 'en-US,en;q=0.9,ur;q=0.8,zh-CN;q=0.7,zh;q=0.6',
'cookie': 'brwsr=3546b2d8-4454-11ec-8b5a-dbae802bd5ca; ACID=3e3752b6-a706-4ddf-95a2-ae88083ebe3e; hasACID=true; locGuestData=eyJpbnRlbnQiOiJTSElQUElORyIsInN0b3JlSW50ZW50IjoiUElDS1VQIiwibWVyZ2VGbGFnIjpmYWxzZSwicGlja3VwIjp7Im5vZGVJZCI6IjMwODEiLCJ0aW1lc3RhbXAiOjE2MzY3ODg5MDEyMTB9LCJwb3N0YWxDb2RlIjp7InRpbWVzdGFtcCI6MTYzNjc4ODkwMTIxMCwiYmFzZSI6Ijk1ODI5In0sInZhbGlkYXRlS2V5IjoicHJvZDp2MjozZTM3NTJiNi1hNzA2LTRkZGYtOTVhMi1hZTg4MDgzZWJlM2UifQ==; vtc=T0u0KwKFJGHeU3asnYGQ_w; TBV=7; DL=94066,,,ip,94066,,; TB_Latency_Tracker_100=1; TB_Navigation_Preload_01=1; crumb=2RJK-XnGcnZ8WeLbzMNhC6uSY75Q9sqHRkR8eVTWjyH; tb_sw_supported=true; TB_SFOU-100=1; AID=wmlspartner%3Dimp_150372%3Areflectorid%3Dimp_zl3TTEwhgxyIUNGVPPU0LViWUkGxlaWkqz2KVY0%3Alastupd%3D1638813128173; locDataV3=eyJpbnRlbnQiOiJTSElQUElORyIsInBpY2t1cCI6W3siYnVJZCI6IjAiLCJub2RlSWQiOiIzMDgxIiwiZGlzcGxheU5hbWUiOiJTYWNyYW1lbnRvIFN1cGVyY2VudGVyIiwibm9kZVR5cGUiOiJTVE9SRSIsImFkZHJlc3MiOnsicG9zdGFsQ29kZSI6Ijk1ODI5IiwiYWRkcmVzc0xpbmUxIjoiODkxNSBHZXJiZXIgUm9hZCIsImNpdHkiOiJTYWNyYW1lbnRvIiwic3RhdGUiOiJDQSIsImNvdW50cnkiOiJVUyIsInBvc3RhbENvZGU5IjoiOTU4MjktMDAwMCJ9LCJnZW9Qb2ludCI6eyJsYXRpdHVkZSI6MzguNDgyNjc3LCJsb25naXR1ZGUiOi0xMjEuMzY5MDI2fSwiaXNHbGFzc0VuYWJsZWQiOnRydWUsInNjaGVkdWxlZEVuYWJsZWQiOnRydWUsInVuU2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZX1dLCJkZWxpdmVyeSI6eyJidUlkIjoiMCIsIm5vZGVJZCI6IjMwODEiLCJkaXNwbGF5TmFtZSI6IlNhY3JhbWVudG8gU3VwZXJjZW50ZXIiLCJub2RlVHlwZSI6IlNUT1JFIiwiYWRkcmVzcyI6eyJwb3N0YWxDb2RlIjoiOTU4MjkiLCJhZGRyZXNzTGluZTEiOiI4OTE1IEdlcmJlciBSb2FkIiwiY2l0eSI6IlNhY3JhbWVudG8iLCJzdGF0ZSI6IkNBIiwiY291bnRyeSI6IlVTIiwicG9zdGFsQ29kZTkiOiI5NTgyOS0wMDAwIn0sImdlb1BvaW50Ijp7ImxhdGl0dWRlIjozOC40ODI2NzcsImxvbmdpdHVkZSI6LTEyMS4zNjkwMjZ9LCJpc0dsYXNzRW5hYmxlZCI6dHJ1ZSwic2NoZWR1bGVkRW5hYmxlZCI6dHJ1ZSwidW5TY2hlZHVsZWRFbmFibGVkIjp0cnVlLCJhY2Nlc3NQb2ludHMiOlt7ImFjY2Vzc1R5cGUiOiJERUxJVkVSWV9BRERSRVNTIn1dfSwic2hpcHBpbmdBZGRyZXNzIjp7ImxhdGl0dWRlIjozOC40NzM4LCJsb25naXR1ZGUiOi0xMjEuMzQzOSwicG9zdGFsQ29kZSI6Ijk1ODI5IiwiY2l0eSI6IlNhY3JhbWVudG8iLCJzdGF0ZSI6IkNBIiwiY291bnRyeUNvZGUiOiJVU0EiLCJnaWZ0QWRkcmVzcyI6ZmFsc2V9LCJhc3NvcnRtZW50Ijp7Im5vZGVJZCI6IjMwODEiLCJkaXNwbGF5TmFtZSI6IlNhY3JhbWVudG8gU3VwZXJjZW50ZXIiLCJhY2Nlc3NQb2ludHMiOm51bGwsImludGVudCI6IlBJQ0tVUCIsInNjaGVkdWxlRW5hYmxlZCI6ZmFsc2V9LCJpbnN0b3JlIjpmYWxzZSwicmVmcmVzaEF0IjoxNjM4ODM0NzI4MjkxLCJ2YWxpZGF0ZUtleSI6InByb2Q6djI6M2UzNzUyYjYtYTcwNi00ZGRmLTk1YTItYWU4ODA4M2ViZTNlIn0=; assortmentStoreId=3081; hasLocData=1; akavpau_p2=1638813728~id=0ddec06ac25050953645a4b5d72af4f0; adblocked=true; com.wm.reflector="reflectorid:imp_zl3TTEwhgxyIUNGVPPU0LViWUkGxlaWkqz2KVY0@lastupd:1638813132000@firstcreate:1636788901158"; next-day=null|true|true|null|1638862621; location-data=94066:San Bruno:CA::0:0|21k;;15.22,46y;;16.96,1kf;;19.87,1rc;;23.22,46q;;25.3,2nz;;25.4,2b1;;27.7,4bu;;28.38,2er;;29.12,1o1;;30.14|2|7|1|1xun;16;0;2.44,1xtf;16;1;4.42,1xwj;16;2;7.04,1ygu;16;3;8.47,1xwq;16;4;9.21; TB_DC_Flap_Test=0; bstc=cqPOkDdcHRbOOWnXljFUXU; mobileweb=0; xpa=; xpm=3+1638862621+T0u0KwKFJGHeU3asnYGQ_w~+0; _pxhd=1wGn8vPC/xYO43oIyZQIBmYn28J4J4/ceMd-WLk8e9M7Qyw0ToljNcm1zXiFNAC5tYcUXy88tg3nBqpdAuT5sQ==:dW/JtmJPPeYviWr1RnbGL8Q8fvsAnk/Tr319tYqmCEF-ZACrf9lbi8vQzvzKY6lDVRH5k5dW2zRPEnPwSFdhe9v-Q1NOMHusjxvPvQ2BFeM=; ak_bmsc=DEAF5D13B7E5DB9D1445A42C981F72DB~000000000000000000000000000000~YAAQP54QApQE/i99AQAATg3Tkw6GGZisOY9WqflSzzYhEOPbRvefmAkpiEYorVwUfg9UEAFnLY8StjWnjYCXsEqzFDDy gnGpbydgZS 20l VJJigKU51o2xuF8AJrgY5QzJhLMA/i9MxW5MRL n0zV BC1PLTb1hhelYx8GmmyDV HifGBSgErgNtb6pUA5ydONX9EprYpZknQfqP30OmVWTnKkloTrQDkfJcJ0vI P/MEZb8U2molWz/GdGwU rbhcdSfa9oWaLaAp8E/DZjsY6YpmW4fmZNQTrUfa7P2db8EDnbWF1zi8r3D 51o2Hi7bd5Bi 8txdaDdl9YT3 Nr4VofqwkH5iXqOmidTDd/fO6qvtQyO0oau3Vjowa xhn04TYYtlQjsb9A; xptwg=3271574160:209D26905C6B780:55789D4:D6539E2E:8EBD026C:E7FE99FF:; TS01b0be75=01538efd7c206a0dc58225169e0b825676e9c5e453ca7af568232e9be3e84e5a6227c4b728ed7713e869287e81ef2d29326fcd6cab; TS013ed49a=01538efd7c206a0dc58225169e0b825676e9c5e453ca7af568232e9be3e84e5a6227c4b728ed7713e869287e81ef2d29326fcd6cab; bm_mi=01610934A8DDD8546E2A0DEF61A8C91C~xOaPkjLqDDIuv11EPeZ1gIxvBQt6PD6zjA7Gd2qVQfVC6cwGG83ojqAvWCSM6IYwVuSjWY3iSyJNH7YzUX2nmRtIlmJUlrAz5tz3OU9v1zqnhHdq0QcCuMve0SUoKRpLcqWK65ocd9vpQR78SbT7CLWkBDckK4ro0g38t1cdBsBb8OZKF21D M5ZU1pHVEuO3MvvWCWNDByoMpg9KcRVvq/67Rbu6fVBemqGJU3g54VcLF6BgDysyAiM1zrC5dHyOznoyGKalanQZpuh4hQQwg==; bm_sv=FA70A3C0284440D78F7D26E1C9316BD3~G6MyGle3ACdP7loWuTml7iej WW4evzORtt1IKVCT/tduzxQYcbTI2Ti1xyqDnz8l6K0ty4wXsRVwrzTkcnDAzku7y4AVIZ5cKsSt9rThXIHazknmtE9Y0OSWFRz8IdFFrO5uHnXxAPnXTdp97vj7fYje/wT3m GfZhmesPZZNs=',
}
params = (
('query', keyword),
('stores', '5939'),
('cat_id', '0'),
('ps', '24'),
('offset', '24'),
('prg', 'desktop'),
('zipcode', '98006'),
('stateOrProvinceCode', 'WA'),
)
session = requests.session()
r = session.get(
'https://www.walmart.com/store/electrode/api/search',
headers=headers,
params=params
)
print(r.status_code) #returns 200
傳遞運行時 cookie（不起作用）
import requests
headers = {
'authority': 'www.walmart.com',
'pragma': 'no-cache',
'cache-control': 'no-cache',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"',
'dnt': '1',
'sec-ch-ua-mobile': '?0',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36',
'sec-ch-ua-platform': '"macOS"',
'content-type': 'application/json',
'accept': '*/*',
'sec-fetch-site': 'same-origin',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://www.walmart.com/store/5939-bellevue-wa/search?query=tillamook',
'accept-language': 'en-US,en;q=0.9,ur;q=0.8,zh-CN;q=0.7,zh;q=0.6'
}
params = (
('query', keyword),
('stores', '5939'),
('cat_id', '0'),
('ps', '24'),
('offset', '24'),
('prg', 'desktop'),
('zipcode', '98006'),
('stateOrProvinceCode', 'WA'),
)
# Get session cookies
session = requests.session()
r = session.get('https://www.walmart.com/')
cookies = session.cookies
cookies_dictionary = cookies.get_dict()
print(cookies)
r = session.get(
'https://www.walmart.com/store/electrode/api/search',
headers=headers,
params=params,
cookies=cookies # Not working
)
print(r.status_code) # Returns 403
更新
response.cookies生成不同的 cookie
{'TS01b0be75': '01538efd7c044ddd7186ec7b8e9a2c7c3f59c4292535357d3ee83514b723af242e66b7d2d4bb89dca3c3512d765a35854d24b328b7', '_pxhd': 'f2ledR7-gSvtTHbIFTwB89DoCT99Whwim/4tbcl-XKEvqQH1mfG94waAxWYDdlEtAS7YXmgpcH9Wg7XztyrF-w==:DGocCdAUsAwd9NAEPkHiaYyPxwgpVX3GTvmhGfztbMb2T6hqgKWOgSYeW1U0qtgwsseesEz/fHMPnVceRuFkILyXsV9wDt9vKy708YcyTOk=', 'akavpau_p2': '1638865229~id=bd97f18a4cb5fae6b4f4625ef3799a9d'}
uj5u.com熱心網友回復：
根據@furas 在評論中的建議，首先嘗試使用 selenium 從該站點獲取 cookie。然后，您可以在發出獲取請求以獲取所需回應和結果的同時在標頭中使用這些 cookie。我使用以下方法取得了成功：
import time
import requests
from selenium import webdriver
def get_cookies():
with webdriver.Chrome() as driver:
driver.get('https://www.walmart.com/store/5939-bellevue-wa/search?query=tillamook')
time.sleep(10)
driver_cookies = driver.get_cookies()
cookie = {c['name']:c['value'] for c in driver_cookies}
return cookie
headers = {
'accept': '*/*',
'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36',
'referer': 'https://www.walmart.com/store/5939-bellevue-wa/search?query=tillamook',
}
params = {
'query': 'tillamook',
'stores': '5939',
'cat_id': '0',
'ps': '24',
'offset': '24',
'prg': 'desktop',
'zipcode': '98006',
'stateOrProvinceCode': 'WA',
}
with requests.session() as session:
r = session.get(
'https://www.walmart.com/store/electrode/api/search',
headers=headers,
params=params,
cookies=get_cookies()
)
print(r.status_code) # Returns 200
print(r.json())
轉載請註明出處，本文鏈接：https://www.uj5u.com/caozuo/377781.html