添加kward和第二個pers后，scrapy停止作業

添加 kward 腳本后停止輸出任何抓取的資料，它只輸出正常的蜘蛛除錯資料。我完全不知道為什么它會那樣做，看起來整個 parseMain 只是坐在那里，什么也不做。

這是我的代碼：

# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy import Request, Spider


class RynekMainSpider(scrapy.Spider):
    name = "RynekMain"
    start_urls = [
        'https://rynekpierwotny.pl/deweloperzy/?page=1']
    def parse(self, response):
        websites = response.css('div.root')
        for websitep in websites:
            websiteurl = websitep.css('div.rp-l0pkv6 a::attr(href)').get()
            href = websitep.css('li.rp-np9kb1 a::attr(href)').get()
            url = response.urljoin(href)
            yield Request(url, cb_kwargs={'websiteurl': websiteurl}, callback=self.parseMain)
    
    def parseMain(self, response, websiteurl):
   # def parse(self, response):
        for quote in response.css('.rp-y89gny.eboilu01 ul li'):
                address = quote.css('address.rp-o9b83y::text').get(),
                name = quote.css('h2.rp-69f2r4::text').get(),
                href = quote.css('li.rp-np9kb1 a::attr(href)').get(),
                PAGETEST = response.css('a.rp-mmikj9::attr(href)').get()
        yield {
            'address' : address,
            'name' : name,
            'href' : href,
            'PAGETEST' : PAGETEST,
            'websiteurl' : websiteurl
            }
        next_page=response.css('a.rp-mmikj9::attr(href)').get()
        if next_page is not None:
            next_page_link=response.urljoin(next_page)
            yield scrapy.Request(url=next_page_link, callback= self.parse)
    
if __name__ == "__main__":
    process =CrawlerProcess()
    process.crawl(RynekMainSpider)
    process.start()

提前感謝您的幫助。編輯：哦，拍攝我忘了告訴我的代碼應該做什么。基本上決議是從“https://rynekpierwotny.pl/deweloperzy/dom-development-sa-955/”之類的子頁面內部獲取網站網址。雖然 parseMain 正在從主頁“https://rynekpierwotny.pl/deweloperzy/?page=1”獲取所有資料（如地址、名稱）。

# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy import Request, Spider


class RynekMainSpider(scrapy.Spider):
    name = "RynekMain"
    start_urls = [
        'https://rynekpierwotny.pl/deweloperzy/?page=1']
   
    
 
    def parse(self, response):
        for quote in response.css('.rp-y89gny.eboilu01 ul li'):
            yield {
                'address' : quote.css('address.rp-o9b83y::text').get(),
                'name' : quote.css('h2.rp-69f2r4::text').get(),
                'href' : quote.css('li.rp-np9kb1 a::attr(href)').get(),
                'PAGETEST' : response.css('a.rp-mmikj9::attr(href)').get()
            }
        next_page=response.css('a.rp-mmikj9::attr(href)').get()
        if next_page is not None:
            next_page_link=response.urljoin(next_page)
            yield scrapy.Request(url=next_page_link, callback= self.parse)
    
if __name__ == "__main__":
    process =CrawlerProcess()
    process.crawl(RynekMainSpider)
    process.start()

這有效

uj5u.com熱心網友回復：

編輯：

我根據您對要編程執行的操作的注釋進行了一些進一步的調整。它應該按照您現在期望的方式作業。

試試這個：

# -*- coding: utf-8 -*-
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy import Request, Spider


class RynekMainSpider(scrapy.Spider):
    name = "RynekMain"
    start_urls = [
        'https://rynekpierwotny.pl/deweloperzy/?page=1']
    def parse(self, response):
        websites = response.css('div#root')[0]
        PAGETEST = response.xpath('//a[contains(@class,"rp-173nt6g")]/../following-sibling::li').css('a::attr(href)').get()
        for website in websites.css('li.rp-np9kb1'):
            page = website.css('a::attr(href)').get()
            address = website.css('address.rp-o9b83y::text').get()
            name = website.css('h2.rp-69f2r4::text').get()
            params = {
            'address' : address,
            'name' : name,
            'href' : page,
            }
            url  = response.urljoin(page)
            yield Request(url=url, cb_kwargs={'params': params}, callback=self.parseMain)
        yield Request(url=response.urljoin(PAGETEST), callback=self.parse)

    def parseMain(self, response, params=None):
        # print(response.url)
        website = response.css('div.rp-l0pkv6 a::attr(href)').get()
        params['website'] = website
        yield params

if __name__ == "__main__":
    process =CrawlerProcess()
    process.crawl(RynekMainSpider)
    process.start()

標籤：Python python-3.x 网页抓取 刮擦

上一篇：請求模塊未在Python中獲取完整網站

下一篇：在R中抓取大象資料