使用Scrapy處理從Indeed重定向<301>

我正在為 Indeed 構建一個個人爬蟲，主要是為了練習 - 我已經設定了它，以便我在每個頁面中提取每 100 個結果的詳細資訊。通過使用搜索查詢，我有一個城市和作業型別的種子串列，回圈在確實 url 的 f 字串中。我將這些結果存盤為字典，以便在將這些結果讀入 Pandas 時，可以將度數型別作為列獲取。

我的問題是我不斷收到Redirecting (301)，我想那是因為并非所有鏈接都滿足薪水的要求。或者，我已經包括在內，meta={'handle_httpstatus_list': [301]}但無論如何我都沒有得到任何結果。

這是我的刮刀：

class IndeedItem(scrapy.Item):
    job_title = Field(output_processor = TakeFirst())
    salary = Field(output_processor = TakeFirst())
    category = Field(output_processor = TakeFirst())
    company = Field(output_processor = TakeFirst())

class IndeedSpider(scrapy.Spider):
    name = 'indeed'
    max_results_per_city = 1000
    #names = pd.read_csv("indeed_names.csv")
    #degree = pd.read_csv("degree_names2.csv",encoding='unicode_escape')
    names = pd.DataFrame({'names':['London', 'Manchester']})
    degree = pd.DataFrame({'degrees':['degree Finance ￡25','degree Engineering ￡25'], 'degree_type':['Finance', 'Engineering']})
    start_urls = defaultdict(list)
    for city in names.names:
        for qualification,name in zip(degree.degrees, degree.degree_type):
            start_urls[name].append(f'https://uk.indeed.com/jobs?q={qualification},000&l={city}&fromage=7&filter=0&limit=100')

    custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36',
        'DOWNLOAD_DELAY':2
    }

    def start_requests(self):
        for category, url in self.start_urls.items():
            for link in url:
                yield scrapy.Request(
                    link, 
                    callback = self.parse,
                    #meta={'handle_httpstatus_list': [301]},
                    cb_kwargs = {
                        'page_count':0,
                        'category':category 
                }
            )

    def parse(self, response, page_count, category):
        if page_count > 30:
            return
        indeed = response.xpath('//div[@id="mosaic-zone-jobcards"]//div')
        for jobs in indeed:
            loader = ItemLoader(IndeedItem(), selector = jobs)
            loader.add_value('category', category)
            loader.add_xpath('job_title', './/h2[@]/span//text()')
            loader.add_xpath('salary', './/div[@]/span//text()')
            loader.add_xpath('company', './/a/div[@]/div[@]/div[@]/div[@]/table[@]/tbody/tr/td[@]/div[@]/pre/span[@]//text()')
            yield loader.load_item
        
        next_page = response.xpath('//ul[@]/li[5]/a//@href').get()
        page_count  = 1
        if next_page is not None:
            yield response.follow(
                next_page, 
                callback = self.parse,
                cb_kwargs = {
                    'page_count': page_count,
                    'category': category
                }
            )

uj5u.com熱心網友回復：

我沒有任何 301 狀態，但是 start_urls 給了我問題，并且您的 xpath 已關閉

這修復了xpath：

import scrapy
from pandas._libs.internals import defaultdict
from scrapy import Field
from scrapy.loader import ItemLoader
from scrapy.loader.processors import TakeFirst
import pandas as pd


class IndeedItem(scrapy.Item):
    job_title = Field(output_processor=TakeFirst())
    salary = Field(output_processor=TakeFirst())
    category = Field(output_processor=TakeFirst())
    company = Field(output_processor=TakeFirst())


class IndeedSpider(scrapy.Spider):
    name = 'indeed'

    custom_settings = {
        'USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36',
        'DOWNLOAD_DELAY': 2
    }

    max_results_per_city = 1000
    # names = pd.read_csv("indeed_names.csv")
    # degree = pd.read_csv("degree_names2.csv",encoding='unicode_escape')
    names = pd.DataFrame({'names': ['London', 'Manchester']})
    degree = pd.DataFrame({'degrees': ['degree Finance ￡25,000', 'degree Engineering ￡25,000'], 'degree_type': ['Finance', 'Engineering']})

    start_urls = defaultdict(list)

    def start_requests(self):
        for city in self.names.names:
            for qualification, name in zip(self.degree.degrees, self.degree.degree_type):
                self.start_urls[name].append(f'https://uk.indeed.com/jobs?q={qualification}&l={city}&fromage=7&filter=0&limit=100')

        for category, url in self.start_urls.items():
            for link in url:
                yield scrapy.Request(
                    link,
                    callback=self.parse,
                    #meta={'handle_httpstatus_list': [301]},
                    cb_kwargs={
                        'page_count': 0,
                        'category': category
                    }
                )

    def parse(self, response, page_count, category):
        if page_count > 30:
            return
        indeed = response.xpath('//div[@]')
        for jobs in indeed:
            loader = ItemLoader(IndeedItem(), selector=jobs)
            loader.add_value('category', category)
            loader.add_xpath('job_title', './/span[@title]//text()')
            loader.add_xpath('salary', './/div[@]/span//text()')
            loader.add_xpath('company', './/span[@]//text()')
            yield loader.load_item()

        next_page = response.xpath('//ul[@]//li[last()]/a/@href').get()
        page_count  = 1
        if next_page:
            yield response.follow(
                next_page,
                callback=self.parse,
                cb_kwargs={
                    'page_count': page_count,
                    'category': category
                }
            )

如果您可以舉一個重定向網址的示例，我可以嘗試為您提供幫助。

標籤：Python 网页抓取 刮的

上一篇：如何進行JavaScript呼叫以從網站抓取資料？

下一篇：為什么在嘗試訪問HTML表中的前兩列時出現錯誤？