獲取要爬取的URL

爬蟲前期作業

用Pycharm打開專案開始寫爬蟲檔案

1. 欄位檔案items

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class NbaprojectItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # pass
    # 創建欄位的固定格式-->scrapy.Field()
    # 英文名
    engName = scrapy.Field()
    # 中文名
    chName = scrapy.Field()
    # 身高
    height = scrapy.Field()
    # 體重
    weight = scrapy.Field()
    # 國家英文名
    contryEn = scrapy.Field()
    # 國家中文名
    contryCh = scrapy.Field()
    # NBA球齡
    experience = scrapy.Field()
    # 球衣號碼
    jerseyNo = scrapy.Field()
    # 入選年
    draftYear = scrapy.Field()
    # 隊伍英文名
    engTeam = scrapy.Field()
    # 隊伍中文名
    chTeam = scrapy.Field()
    # 位置
    position = scrapy.Field()
    # 東南部
    displayConference = scrapy.Field()
    # 磁區
    division = scrapy.Field()

2. 爬蟲檔案

import scrapy
import json
from nbaProject.items import NbaprojectItem

class NbaspiderSpider(scrapy.Spider):
    name = 'nbaSpider'
    allowed_domains = ['nba.com']
    # 第一次爬取的網址,可以寫多個網址
    # start_urls = ['http://nba.com/']
    start_urls = ['https://china.nba.com/static/data/league/playerlist.json']
    # 處理網址的response
    def parse(self, response):
        # 因為訪問的網站回傳的是json格式,首先用第三方包處理json資料
        data = json.loads(response.text)['payload']['players']
        # 以下串列用來存放不同的欄位
        # 英文名
        engName = []
        # 中文名
        chName = []
        # 身高
        height = []
        # 體重
        weight = []
        # 國家英文名
        contryEn = []
        # 國家中文名
        contryCh = []
        # NBA球齡
        experience = []
        # 球衣號碼
        jerseyNo = []
        # 入選年
        draftYear = []
        # 隊伍英文名
        engTeam = []
        # 隊伍中文名
        chTeam = []
        # 位置
        position = []
        # 東南部
        displayConference = []
        # 磁區
        division = []
        # 計數
        count = 1
        for i in data:
            # 英文名
            engName.append(str(i['playerProfile']['firstNameEn'] + i['playerProfile']['lastNameEn']))
            # 中文名
            chName.append(str(i['playerProfile']['firstName'] + i['playerProfile']['lastName']))
            # 國家英文名
            contryEn.append(str(i['playerProfile']['countryEn']))
            # 國家中文
            contryCh.append(str(i['playerProfile']['country']))
            # 身高
            height.append(str(i['playerProfile']['height']))
            # 體重
            weight.append(str(i['playerProfile']['weight']))
            # NBA球齡
            experience.append(str(i['playerProfile']['experience']))
            # 球衣號碼
            jerseyNo.append(str(i['playerProfile']['jerseyNo']))
            # 入選年
            draftYear.append(str(i['playerProfile']['draftYear']))
            # 隊伍英文名
            engTeam.append(str(i['teamProfile']['code']))
            # 隊伍中文名
            chTeam.append(str(i['teamProfile']['displayAbbr']))
            # 位置
            position.append(str(i['playerProfile']['position']))
            # 東南部
            displayConference.append(str(i['teamProfile']['displayConference']))
            # 磁區
            division.append(str(i['teamProfile']['division']))

            # 創建item欄位物件,用來存盤資訊 這里的item就是對應上面導的NbaprojectItem
            item = NbaprojectItem()
            item['engName'] = str(i['playerProfile']['firstNameEn'] + i['playerProfile']['lastNameEn'])
            item['chName'] = str(i['playerProfile']['firstName'] + i['playerProfile']['lastName'])
            item['contryEn'] = str(i['playerProfile']['countryEn'])
            item['contryCh'] = str(i['playerProfile']['country'])
            item['height'] = str(i['playerProfile']['height'])
            item['weight'] = str(i['playerProfile']['weight'])
            item['experience'] = str(i['playerProfile']['experience'])
            item['jerseyNo'] = str(i['playerProfile']['jerseyNo'])
            item['draftYear'] = str(i['playerProfile']['draftYear'])
            item['engTeam'] = str(i['teamProfile']['code'])
            item['chTeam'] = str(i['teamProfile']['displayAbbr'])
            item['position'] = str(i['playerProfile']['position'])
            item['displayConference'] = str(i['teamProfile']['displayConference'])
            item['division'] = str(i['teamProfile']['division'])
            # 列印爬取資訊
            print("傳輸了",count,"條欄位")
            count += 1
            # 將欄位交回給引擎 -> 管道檔案
            yield item

3. 組態檔->開啟管道檔案
   

# Scrapy settings for nbaProject project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# ----------不做修改部分---------
BOT_NAME = 'nbaProject'

SPIDER_MODULES = ['nbaProject.spiders']
NEWSPIDER_MODULE = 'nbaProject.spiders'
# ----------不做修改部分---------

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'nbaProject (+http://www.yourdomain.com)'

# Obey robots.txt rules
# ----------修改部分(可以自行查這是啥東西)---------
# ROBOTSTXT_OBEY = True
# ----------修改部分---------

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'nbaProject.middlewares.NbaprojectSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'nbaProject.middlewares.NbaprojectDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# 開啟管道檔案
# ----------修改部分---------
ITEM_PIPELINES = {
   'nbaProject.pipelines.NbaprojectPipeline': 300,
}
# ----------修改部分---------
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

4. 管道檔案 -> 將欄位寫進mysql

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

import pymysql
class NbaprojectPipeline:
	# 初始化函式
    def __init__(self):
        # 連接資料庫 注意修改資料庫資訊
        self.connect = pymysql.connect(host='域名', user='用戶名', passwd='密碼',
                                       db='資料庫', port=埠號) 
        # 獲取游標
        self.cursor = self.connect.cursor()
        # 創建一個表用于存放item欄位的資料
        createTableSql = """
                            create table if not exists `nbaPlayer`(
                            playerId INT UNSIGNED AUTO_INCREMENT,
                            engName varchar(80),
                            chName varchar(20),
                            height varchar(20),
                            weight varchar(20),
                            contryEn varchar(50),
                            contryCh varchar(20),
                            experience int,
                            jerseyNo int,
                            draftYear int,
                            engTeam varchar(50),
                            chTeam varchar(50),
                            position varchar(50),
                            displayConference varchar(50),
                            division varchar(50),
                            primary key(playerId)
                            )charset=utf8;
                            """
        # 執行sql陳述句
        self.cursor.execute(createTableSql)
        self.connect.commit()
        print("完成了創建表的作業")
	#每次yield回來的欄位會在這里做處理
    def process_item(self, item, spider):
    	# 列印item增加觀賞性
    	print(item)
        # sql陳述句
        insert_sql = """
        insert into nbaPlayer(
        playerId, engName, 
        chName,height,
        weight,contryEn,
        contryCh,experience,
        jerseyNo,draftYear
        ,engTeam,chTeam,
        position,displayConference,
        division
        ) VALUES (null,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
        # 執行插入資料到資料庫操作
        # 引數(sql陳述句,用item欄位里的內容替換sql陳述句的占位符)
        self.cursor.execute(insert_sql, (item['engName'], item['chName'], item['height'], item['weight']
                                         , item['contryEn'], item['contryCh'], item['experience'], item['jerseyNo'],
                                         item['draftYear'], item['engTeam'], item['chTeam'], item['position'],
                                         item['displayConference'], item['division']))
        # 提交，不進行提交無法保存到資料庫
        self.connect.commit()
        print("資料提交成功!")

啟動爬蟲

螢屏上滾動的資料

去資料庫查看資料

簡簡單單就把球員資料爬回來啦~

原創不易,請給博主一個小小的贊吧~