from scrapy import Spider
from scrapy.http import Request
class AuthorSpider(Spider):
name = 'book'
start_urls = ['https://www.amazon.com/s?k=school bags&rh=n:1069242&ref=nb_sb_noss']
def parse(self, response):
books = response.xpath("//h2/a/@href").extract()
for book in books:
url = response.urljoin(book)
yield Request(url, callback=self.parse_book)
def parse_book(self, response):
table=response.xpath("//table[@id='productDetails_detailBullets_sections1']").extract_first()
yield{
't':table
}
我試圖刮桌子，但我不知道如何text從桌子中提取試圖刮product information這是我提取桌子的鏈接
uj5u.com熱心網友回復：
要抓取表，您可以遍歷表頭和表資料并將它們分配給鍵和值，然后生成完整的字典。請參閱下面的示例
from scrapy import Spider
from scrapy.http import Request
class AuthorSpider(Spider):
name = 'book'
start_urls = ['https://www.amazon.com/s?k=school bags&rh=n:1069242&ref=nb_sb_noss']
def parse(self, response):
books = response.xpath("//h2/a/@href").extract()
for book in books:
url = response.urljoin(book)
yield Request(url, callback=self.parse_book)
def parse_book(self, response):
details = {}
for product_detail in response.xpath("//*[contains(@id,'productDetails')]//table/tr"):
key = product_detail.xpath("normalize-space(./th/text())").get()
value = product_detail.xpath("normalize-space(./td/text())").get().replace("\u200e", "")
if "best sellers rank" in key.lower():
det_list = product_detail.xpath("./td/descendant::*/text()").getall()
value = "".join([i.strip() for i in det_list])
if "customer reviews" in key.lower():
det_list = product_detail.xpath("./td/descendant::span/text()").getall()
value = " ".join([i.strip() for i in det_list])
details[key] = value
yield details
uj5u.com熱心網友回復：
它僅適用于您在問題中遇到的這種表：
from scrapy import Spider
from scrapy.http import Request
class AuthorSpider(Spider):
name = 'book'
start_urls = ['https://www.amazon.com/s?k=school bags&rh=n:1069242&ref=nb_sb_noss']
def parse(self, response):
books = response.xpath("//h2/a/@href").extract()
for book in books:
url = response.urljoin(book)
# just for the example
url='https://www.amazon.com/Piel-Leather-Double-Flap-Over-Backpack/dp/B00GNEY85A/ref=sr_1_1_sspa?keywords=school+bags&qid=1642846253&s=office-products&sr=1-1-spons&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUExMkdMT1hKSkI1UVFTJmVuY3J5cHRlZElkPUEwNTQxMDA5M0c1R0xRQVUwTVdKViZlbmNyeXB0ZWRBZElkPUEwNzc5Njc4MUdQR09VMVBGSTlGSSZ3aWRnZXROYW1lPXNwX2F0ZiZhY3Rpb249Y2xpY2tSZWRpcmVjdCZkb05vdExvZ0NsaWNrPXRydWU&th=1'
yield Request(url, callback=self.parse_book)
def parse_book(self, response):
rows = response.xpath('//div[@id="prodDetails"]//tr')
table = {}
for row in rows:
key = row.xpath('.//th//text()').get(default='').strip()
# this will work for most of the rows (except "Customer Reviews" and "Best Sellers Rank"):
# value = line.xpath('.//td//text()').get(default='').strip()
# this will work for all the rows
value = row.xpath('.//td/text() | .//td//span/text()').getall()
value = ''.join(value).strip()
table.update({key: value})
yield table
這只是一個例子。您需要檢查您可以獲得的不同型別的表并相應地調整您的代碼。
我們正在做的是逐行瀏覽表格并從中提取文本，然后將其添加到字典并最終生成它。
要獲取我們正在使用的文本/text()。搜索xpath cheat sheet它會幫助你。
轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/418824.html
標籤：
上一篇：我無法使用bs4訪問某些元素？