scrapy使用selenium爬取豆瓣閱讀-有解無憂

首先創建scrapy專案
命令：scrapy startproject douban_read
創建spider
命令：scrapy genspider douban_spider url
網址：https://read.douban.com/charts

關鍵注釋代碼中有，若有不足，請多指教

scrapy專案目錄結構如下
在這里插入圖片描述
douban_spider.py檔案代碼
爬蟲檔案

import scrapy
import re, json

from ..items import DoubanReadItem


class DoubanSpiderSpider(scrapy.Spider):
    name = 'douban_spider'
    # allowed_domains = ['www']
    start_urls = ['https://read.douban.com/charts']

    def parse(self, response):
        # print(response.text)
        # 獲取圖書分類的url
        type_urls = response.xpath('//div[@class="rankings-nav"]/a[position()>1]/@href').extract()
        # print(type_urls)
        for type_url in type_urls:
            # /charts?type=unfinished_column&index=featured&dcs=charts&dcm=charts-nav
            part_param = re.search(r'charts\?(.*?)&dcs', type_url).group(1)
            # https://read.douban.com/j/index//charts?type=intermediate_finalized&index=science_fiction&verbose=1
            ajax_url = 'https://read.douban.com/j/index//charts?{}&verbose=1'.format(part_param)
            yield scrapy.Request(ajax_url, callback=self.parse_ajax, encoding='utf-8', meta={'request_type': 'ajax'})

    def parse_ajax(self, response):

        # print(response.text)
        # 獲取分類中圖書的json資料
        json_data = json.loads(response.text)
        for data in json_data['list']:
            item = DoubanReadItem()
            item['book_id'] = data['works']['id']
            item['book_url'] = data['works']['url']
            item['book_title'] = data['works']['title']
            item['book_author'] = data['works']['author']
            item['book_cover_image'] = data['works']['cover']
            item['book_abstract'] = data['works']['abstract']
            item['book_wordCount'] = data['works']['wordCount']
            item['book_kinds'] = data['works']['kinds']
            # 把item yield給Itempipeline
            yield item

item.py檔案代碼
專案的目標檔案

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class DoubanReadItem(scrapy.Item):
    # define the fields for your item here like:
    book_id = scrapy.Field()
    book_url = scrapy.Field()
    book_title = scrapy.Field()
    book_author = scrapy.Field()
    book_cover_image = scrapy.Field()
    book_abstract = scrapy.Field()
    book_wordCount = scrapy.Field()
    book_kinds = scrapy.Field()

my_download_middle.py檔案代碼
所有request都會經過下載中間件，可以通過定制中間件，來完成設定代理，動態設定請求頭，自定義下載等操作

import random
import time
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from scrapy.http.response.html import HtmlResponse


class MymiddleWares(object):
    def __init__(self):
        # 請求頭串列
        self.USER_AGENT_LIST = [
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
            "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
            "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
        ]

    def process_request(self, request, spider):
        '''
        下載中間件處理requests的方法
        :param request:馬上要被下載器下載request
        :param spider:
        :return:
        '''
        # 在spider中設定了meta的request_type的值為ajax  meta引數會貫穿整個scrapy
        request_type = request.meta.get('request_type')
        # 如果不是ajax請求就需要通過selenium來自定義下載request
        if not request_type:
            print('in middler')
            # 1、創建driver
            driver = webdriver.Chrome()
            # 2、請求url
            driver.get(request.url)
            # 3、等待
            # driver.implicitly_wait(20)
            time.sleep(3)
            # 4、獲取頁面內容
            html_str = driver.page_source
            # 直接回傳HtmlResponse給spider決議  下載器就不會下載這個request 達到自定義下載的目的
            return HtmlResponse(url=request.url, body=html_str, request=request, encoding='utf-8')

        else:
            # 如果是ajax請求就需要通過scrapy下載器來下載request
            # ajax請求直接回傳json資料不適合上面的selenium下載
            ua = random.choice(self.USER_AGENT_LIST)
            # 設定請求頭
            if ua:
                request.headers.setdefault('User-Agent', ua)
            request.headers.setdefault('X-Requested-With', 'XMLHttpRequest')

pipeline.py檔案代碼
專案的管道檔案

import pymongo
from itemadapter import ItemAdapter


class MongoPipeline:
    # 存盤集合名字
    collection_name = 'book'

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
        )

    def open_spider(self, spider):
        '''
        當spider啟動的時候呼叫
        :param spider:
        :return:
        '''
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    # 保存到mongo的douban_read資料庫下的book集合中
    def process_item(self, item, spider):
        self.db[self.collection_name].update({'book_id': item['book_id']}, {'$set': dict(item)}, True)
        # True：有則修改 無則新增
        print(item)
        return item

settings.py檔案代碼
配置資訊

 # Scrapy settings for douban_read project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'douban_read'

SPIDER_MODULES = ['douban_read.spiders']
NEWSPIDER_MODULE = 'douban_read.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'douban_read (+http://www.yourdomain.com)'

# Obey robots.txt rules
# robot協議
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
# 默認請求頭
DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
  'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36',

}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'douban_read.middlewares.DoubanReadSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# 配置下載器中間件
DOWNLOADER_MIDDLEWARES = {
   'douban_read.my_download_middle.MymiddleWares': 543,
}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# 配置ITEM_PIPELINES
ITEM_PIPELINES = {
   'douban_read.pipelines.MongoPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# 配置mongo
MONGO_URI = 'localhost'
# 創建資料庫：douban_read
MONGO_DATABASE = 'douban_read'

最后啟動該專案即可
scrapy crawl douban_spider

資料就保存到mongo資料庫了
在這里插入圖片描述

轉載請註明出處，本文鏈接：https://www.uj5u.com/qianduan/81593.html

標籤：其他

上一篇：python爬蟲練習爬取美團網酒店資訊

下一篇：Python爬蟲爬取網站資訊的基本流程