cmd 命令創建專案
scrapy startproject yiyaowang
cd yiyaowang
scrapy genspider yaowang yaowang.com
先進入settings.py檔案將服從爬蟲協議改成False，因為有些網站不蓋爬取不了，因此都改了
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
items.py
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class YiyaowangItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
# 定義藥品名
title = scrapy.Field()
# 定義藥品價格
price = scrapy.Field()
# 定義評論數
comment = scrapy.Field()
pass
yaowang.py
# -*- coding: utf-8 -*-
import scrapy
from ..items import YiyaowangItem
class YaowangSpider(scrapy.Spider):
name = 'yaowang'
# allowed_domains = ['yaowang.com']
# 分頁：找URL規律
base_url = 'https://www.111.com.cn/categories/953710-j{}.html'
start_urls = []
for i in range(1,51):
start_urls.append(base_url.format(i))
def parse(self, response):
# 實體化物件
item = YiyaowangItem()
# 提取資料
li_list = response.xpath('//ul[@id="itemSearchList"]/li')
for li in li_list:
# 獲取藥品名
title = li.xpath('.//p[@class="titleBox"]/a/text()').extract()[1].strip()
# 發現問題：一片空白
# 分析：
# 1. xpath路徑問題
# 2. 使用xpath獲取值的時候，串列中的第一個元素是空白字符
# 解決：
# 使用extract()或者getall()獲取串列，取出我們想要的資料即可
# 獲取藥品價格
# price = li.xpath('.//p[@class="price"]/span/text()').extract_first().strip()
# 發現問題：
# 有的藥品有價格，有的價格為None
# 經過在頁面中的查看，發現，價格為None的藥品，其實是有真實價格的
# 所以，斷定，xpath路徑有問題，爬蟲爬取的是網頁源代碼，我們看網頁源代碼，價格的span外面還有一側標簽
price = li.xpath('.//p[@class="price"]//span/text()').extract_first().strip()
# 獲取評論數
comment = li.xpath('.//a[@id="pdlink3"]/em/text()').get()
item['title'] = title
item['price'] = price
item['comment'] = comment
yield item
# print(title,price,comment)
pass
pipelines.py
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
# 保存MongoDB資料庫
import pymongo
class YiyaowangPipeline(object):
def open_spider(self,spider):
# 1. 鏈接資料庫
self.client = pymongo.MongoClient(host='10.10.34.163',port=27017)
# 2. 進入資料庫
self.db = self.client['yiyaowang']
# 3. 進入集合
self.col = self.db['yaowang']
pass
def process_item(self, item, spider):
# 插入資料
self.col.insert(dict(item))
return item
# 關閉資料庫
def close_spider(self,spider):
self.client.close()
最后記得將settings.py 里面ITEM_PIPELINES注釋解開，不解開的話是不會執行pipelinnes檔案的
ITEM_PIPELINES = {
'yiyaowang.pipelines.YiyaowangPipeline': 300,
}
轉載請註明出處，本文鏈接：https://www.uj5u.com/houduan/232049.html
標籤：python