試圖抓取網頁的內容-有解無憂

我想獲取此網頁的單詞表：https ://10fastfingers.com/typing-test/english （您應該輸入的單詞串列），我已經設法從 HTML 中獲取了單詞表，但我只是可以' t 得到它的內容。有人可以幫助我嗎？謝謝

這是我的代碼：

from bs4 import BeautifulSoup
from urllib.request import urlopen , Request
import requests

url = "https://10fastfingers.com/typing-test/german"
page = urlopen(Request(url, headers={'User-Agent': 'Mozilla'})).read()
# page = requests.get(url).text
soup = BeautifulSoup(page, "lxml")

match = soup.find("div", class_="row main-layout")
wordlist = match.find("div",  {'id': 'wordlist'})
with open("page content.txt", "w") as file:
    file.write(str(page))
print(wordlist)

uj5u.com熱心網友回復：

您不會從執行 HTML 請求中獲取單詞，因為網站服務器不會將它們存盤在 HTML 源檔案中。事實證明，它通過在加載 HTML 后發出 POST 請求來動態生成單詞。

POST 請求發送到 url：https ://10fastfingers.com/speedtests/get_words如果您在瀏覽器的開發人員工具中打開“網路”選項卡并重繪，您可以準確地看到您自己的請求標頭、cookie 和資料是什么這頁紙。

這是我撰寫的一些示例代碼，用于在 Python 而不是瀏覽器中執行相同的 POST 請求：

# import brotli
import requests
from pprint import pprint

url = "https://10fastfingers.com/speedtests/get_words"

data = {
    "speedtest_mode": "",
    "speedtest_id": "1",
}

cookies = {
    "CAKEPHP": "af57ps7vtafsa3firr35453gtb",
    "CookieConsent": "{stamp:'ojcUTA3IjOXHgdsGXA9u9toM7iRROW28zzy1o3KlbxZssbrOj3a81Q==',necessary:true,preferences:true,statistics:true,marketing:true,ver:2,utc:1654954695891,region:'us'}",
    "CakeCookie[lang]": "Q2FrZQ==.5exP",
    "CakeCookie[alternate_language_suggestion]": "Q2FrZQ==.9PBdWA==",
}

headers = {
    "accept": "*/*",
    "accept-encoding": "gzip, deflate, br",
    "accept-language": "en-US,en;q=0.9",
    "content-length": "30",
    "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
    "cookie": "CAKEPHP=af57ps7vtafsa3firr35453gtb; CookieConsent={stamp:'ojcUTA3IjOXHgdsGXA9u9toM7iRROW28zzy1o3KlbxZssbrOj3a81Q==',necessary:true,preferences:true,statistics:true,marketing:true,ver:2,utc:1654954695891,region:'us'}; CakeCookie[lang]=Q2FrZQ==.5exP; CakeCookie[alternate_language_suggestion]=Q2FrZQ==.9PBdWA==",
    "dnt": "1",
    "origin": "https://10fastfingers.com",
    "referer": "https://10fastfingers.com/typing-test/english",
    "sec-fetch-dest": "empty",
    "sec-fetch-mode": "cors",
    "sec-fetch-site": "same-origin",
    "sec-gpc": "1",
    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.115 Safari/537.36",
    "x-requested-with": "XMLHttpRequest",
}

response_object = requests.post(url, data=data, cookies=cookies, headers=headers)
print("vvv APPARENT ENCODING vvv")
print(response_object.apparent_encoding)
print()
print("vvv ENCODING vvv")
print(response_object.encoding)
print()
print("vvv HEADERS vvv")
pprint(vars(response_object.headers))
print()
print("vvv RESPONSE TEXT vvv")
print(response_object.text)
print()
# print("vvv BROTLI DECODED CONTENT vvv")
# print(brotli.decompress(response_object.content))

...以及列印陳述句的結果：

vvv APPARENT ENCODING vvv
ascii

vvv ENCODING vvv
UTF-8

vvv HEADERS vvv
{'_store': OrderedDict([('date', ('Date', 'Sat, 11 Jun 2022 14:40:21 GMT')),
                        ('content-type',
                         ('Content-Type', 'text/html; charset=UTF-8')),
                        ('transfer-encoding', ('Transfer-Encoding', 'chunked')),
                        ('connection', ('Connection', 'keep-alive')),
                        ('vary', ('vary', 'Accept-Encoding')),
                        ('x-cache', ('x-cache', 'Miss from cloudfront')),
                        ('via',
                         ('via',
                          '1.1 af9b5a8e96971e0d2d7c6fed1b8873b2.cloudfront.net '
                          '(CloudFront)')),
                        ('x-amz-cf-pop', ('x-amz-cf-pop', 'MIA3-P4')),
                        ('x-amz-cf-id',
                         ('x-amz-cf-id',
                          'sMSHv1sVN869z-CZrV2R9q46zGssFwR_tk2vjc7_8Wmcy9zBCuryIg==')),
                        ('cf-cache-status', ('CF-Cache-Status', 'DYNAMIC')),
                        ('expect-ct',
                         ('Expect-CT',
                          'max-age=604800, '
                          'report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"')),
                        ('report-to',
                         ('Report-To',
                          '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report\\/v3?s=GU1vlFpOWgC+Qg9aQZKz2TJGr6bSAoaytdXlGkSdINz6YsKN/nbJg9OkQx1HzzKhzCRHLaGZOaVIaLLQr00u6Z2m3sJnXGOzIbyu1uFhk7SkDtlbxlHBMFNhS++De/R1f0Mo"}],"group":"cf-nel","max_age":604800}')),
                        ('nel',
                         ('NEL',
                          '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}')),
                        ('server', ('Server', 'cloudflare')),
                        ('cf-ray', ('CF-RAY', '719b19f31ad6220f-MIA')),
                        ('content-encoding', ('Content-Encoding', 'br'))])}

vvv RESPONSE TEXT vvv
and|put|ask|own|the|much|come|work|grow|some|along|other|near|letter|we|soon|play|new|always|few|until|their|once|water|there|home|both|story|saw|enough|tree|not|well|tell|you|sentence|need|add|below|all|family|quick|so|our|who|or|which|around|letter|hear|where|then|she|paper|our|back|miss|it|mean|too|add|young|mother|on|almost|to|now|had|four|men|never|got|end|year|two|in|move|state|river|because|even|line|down|while|did|has|house|do|stop|about|large|at|paper|above|it's|help|not|between|no|question|turn|watch|next|only|as|earth|school|family|place|and|animal|any|change|house|here|change|begin|point|would|and|until|about|run|song|up|never|leave|walk|often|much|tell|go|food|are|again|example|quite|is|man|away|feet|of|some|name|few|always|start|ask|learn|small|be|many|first|hand|far|great|idea|no|good|why|you|side|children|night|know|hand|name|see|those|thought|as|how|picture|does|country|almost|new|time|than|grow|way|with|always|children|do|made|eye|below|quickly|know|before|sound|year|later|after|open|Indian|never|every|hear|sometimes|your|got|on|sound|first|run|book|only|paper|even|want|for|each|are|him|they|stop|may|three|me|where|about|is|go|ask|took|small|in|said|far|set|does|just|went|came|different|other|hard|us|line|such|what|soon|really|came|run|when|watch|why|this|not|but|does|live|mean|white|miss|but|then|which|tree|second|being|quick|something|find|big|together|his|up|get|home|over|school|must|idea|light|he|river|those|more|often|are|page|don't|last|day|through|over|work|three|still|people|tree|enough|near|often|try|side|around|near|be|was|let|these|oil|around|year|end|know|got|country|give|America|America|world|these|cut|learn|them|add|might|hear|being|of|they|change|boy|night|girl|it|high|such|said|say|eye|too|Indian|may|our|she|sentence|should|her|like|more|day|car|live|important|found|kind|boy|find|how|far|side|keep|start|if|who|mean|hard|in|both|close|them|will|land|book|page|have|really|kind|show|use|away|again|little|give|four|back|he|said|most|wri

請注意，回應內容是使用 brotli 壓縮編碼的，因此您的腳本需要識別表觀編碼和實際編碼。早些時候，在運行腳本時，由于某種原因，它無法從回應中識別出任何編碼，所以我收到了胡言亂語，我必須用 brotli 解壓縮才能得到實際的單詞......

uj5u.com熱心網友回復：

這個答案是從@hmomin 的真棒答案修改而來的，除了這個答案按照您的要求使用德語而不是英語：

import requests

url = "https://10fastfingers.com/speedtests/get_words"

data = {
    "speedtest_mode": "",
    "speedtest_id": "2",
}

cookies = {
    "CAKEPHP": "c79f210mdl6n2jpp962g9vjk9f",
    "CookieConsent": "{stamp:'6CPboWb OVPYxb8oz6CfbjxqXJzoYqXEDiPE62FxLheS24AUkkmksA==',necessary:true,preferences:true,statistics:true,marketing:true,ver:2,utc:1654956041620,region:'us'}",
    "CakeCookie[lang]": "Q2FrZQ==.5exP",
    "CakeCookie[alternate_language_suggestion]": "Q2FrZQ==.9PBdWA==",
}

headers = {
    "accept": "*/*",
    "accept-encoding": "gzip, deflate, br",
    "accept-language": "en-US,en;q=0.9",
    "cache-control": "no-cache",
    "content-length": "30",
    "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
    "cookie": "CAKEPHP=c79f210mdl6n2jpp962g9vjk9f; CookieConsent={stamp:'6CPboWb OVPYxb8oz6CfbjxqXJzoYqXEDiPE62FxLheS24AUkkmksA==',necessary:true,preferences:true,statistics:true,marketing:true,ver:2,utc:1654956041620,region:'us'}; CakeCookie[lang]=Q2FrZQ==.5exP; CakeCookie[alternate_language_suggestion]=Q2FrZQ==.9PBdWA==",
    "dnt": "1",
    "origin": "https://10fastfingers.com",
    "pragma": "no-cache",
    "referer": "https://10fastfingers.com/typing-test/german",
    "sec-fetch-dest": "empty",
    "sec-fetch-mode": "cors",
    "sec-fetch-site": "same-origin",
    "sec-gpc": "1",
    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.99 Safari/537.36",
    "x-requested-with": "XMLHttpRequest",
}

response_object = requests.post(url, data=data, cookies=cookies, headers=headers)
words = response_object.text
print(words)

uj5u.com熱心網友回復：

該網站使用 ajax 呈現內容，您可以使用此決議獲取內容，這取決于您:)

import  requests
headers = {
    'authority': '10fastfingers.com',
    'accept': '*/*',
    'accept-language': 'en,en-US;q=0.9,ar;q=0.8',
    # 'content-length': '0',
    # Requests sorts cookies= alphabetically
    # 'cookie': 'CAKEPHP=frcdkr9lui4ceuktt7imo0c7f0; _ga=GA1.2.365908205.1654969329; _gid=GA1.2.518644676.1654969329; CookieConsent={stamp:'XCkdg91rCyjPGD5k/I6ib24LWrg88uqVGZ17NwZ4unZU1JRQIxBfmw==',necessary:true,preferences:true,statistics:true,marketing:true,ver:2,utc:1654969345495,region:'ma'}; CakeCookie[lang]=Q2FrZQ==.5exP; CakeCookie[alternate_language_suggestion]=Q2FrZQ==.9PBdWA==; _gat=1',
    'dnt': '1',
    'origin': 'https://10fastfingers.com',
    'referer': 'https://10fastfingers.com/typing-test/english',
    'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="102", "Google Chrome";v="102"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'sec-fetch-dest': 'empty',
    'sec-fetch-mode': 'cors',
    'sec-fetch-site': 'same-origin',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',
    'x-requested-with': 'XMLHttpRequest',
}
Top_Ranking_url = 'https://10fastfingers.com/speedtests/render_highscore_get_top_ranking/1/1'
Tests_taken_url = "https://10fastfingers.com/speedtests/render_highscore_get_tests_taken"
Global_Challenge_url ="https://10fastfingers.com/speedtests/render_highscore_get_global_challenge"
response = requests.post(Top_Ranking_url, headers=headers)
print(response.content)```

轉載請註明出處，本文鏈接：https://www.uj5u.com/shujuku/489018.html

標籤：Python html 网页抓取屏幕抓取

上一篇：抓取一個字串并將其轉換為整數

下一篇：使用Python進行Selenium報廢中的FindAll