北郵 python 爬蟲爬取鏈家的新房資料進行資料處理

博主宣告：用途僅供學習

items.py

import scrapy


class MyItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()    # 名稱
    place1 = scrapy.Field()   # 地理位置
    place2 = scrapy.Field()
    place3 = scrapy.Field()
    model = scrapy.Field()   # 房型
    aera = scrapy.Field()   # 面積
    totalprice = scrapy.Field()   # 總價
    UnitPrice = scrapy.Field()    # 單價
    unit = scrapy.Field()    # 價格單位

spider.py

import scrapy
from linajia.items import MyItem  # 從items.py中引入MyItem物件

class mySpider(scrapy.spiders.Spider):
    name = "linajia"  # 爬蟲的名字是linajia
    allowed_domains = ["bj.lianjia.com/"]  # 允許爬取的網站域名
    start_urls = ["https://bj.fang.lianjia.com/loupan/"]
    # 多頁爬取
    for pg in range(2, 20):
        start_urls.append("https://bj.fang.lianjia.com/loupan/pg{}/".format(pg))
    # 減慢爬蟲速度,保證順序不亂序
    download_delay = 1

    def parse(self, response):  # 決議爬取的內容
        item = MyItem()  # 生成一個在items.py中定義好的Myitem物件,用于接收爬取的資料
        for each in response.xpath('/html/body/div[4]/ul[2]/li'):
            try:
                item['name'] = each.xpath("div/div[1]/a/text()").extract()[0]
                item['place1'] = each.xpath("div/div[2]/span[1]/text()").extract()[0]
                item['place2'] = each.xpath("div/div[2]/span[2]/text()").extract()[0]
                item['place3'] = each.xpath("div/div[2]/a/text()").extract()[0]
                #  取最小戶型
                l = each.xpath("div/a/span[1]/text()").extract()
                if len(l) == 0:  # 最小戶型的資料可能不存在，進行判斷，如果不存在，那么賦值為''
                    item['model'] = ''
                else:
                    item['model'] = l[0]
                # item['aera']取最小面積
                l1 = each.xpath("div/div[3]/span/text()").extract()
                if len(l1):   # 最小面積的資料存在時，進行提取最小值
                    str = l1[0]
                    startpos = str.find(" ") + 1
                    endpos = str.find("-")
                    if endpos == -1:
                        endpos = str.find("m")
                    item['aera'] = str[startpos: endpos]
                else:   # 最小面積不存在時，賦值為空串''
                    item['aera'] = ''
                # item['totalprice']
                l2 = each.xpath("div/div[6]/div[2]/text()").extract()
                # item['UnitPrice']
                l3 = each.xpath("div/div[6]/div[1]/span[1]/text()").extract()
                unit = each.xpath("div/div[6]/div/span[2]/text()").extract()

                # 由于存在網頁顯示均值的位置可能出現總價，那么進行如果進行不處理讀取，會導致某些行的資料
                # 在均值的位置顯示總價，而總價的位置顯示為空
                if -1 != unit[0].find("總價"):
                    item['totalprice'] = l3[0]   # 將均值處顯示的總價放置于總價的位置
                    item['UnitPrice'] = ''
                else:
                    if len(l3) == 0:
                        item['UnitPrice'] = ''
                    else:
                        item['UnitPrice'] = l3[0]
                    if len(l2) == 0:
                        item['totalprice'] = ''
                    else:
                        item['totalprice'] = l2[0]
                yield item
            except ValueError:
                pass

DataProcess.py

import numpy as np
import pandas as pd

# 打開CSV檔案
fileNameStr = 'MyData.csv'
orig_df = pd.read_csv(fileNameStr, encoding='gbk', dtype=str)

# 1.將字串的列前后空格去掉
orig_df['name'] = orig_df['name'].str.strip()
orig_df['place1'] = orig_df['place1'].str.strip()
orig_df['place2'] = orig_df['place2'].str.strip()
orig_df['place3'] = orig_df['place3'].str.strip()
orig_df['model'] = orig_df['model'].str.strip()
orig_df['aera'] = orig_df['aera'].str.strip()
orig_df['totalprice'] = orig_df['totalprice'].str.strip()
orig_df['UnitPrice'] = orig_df['UnitPrice'].str.strip()

# 2.將aera變為整型
orig_df['aera'] = orig_df['aera'].fillna(0).astype(np.int)

# 3.將單價變為整型
orig_df['UnitPrice'] = orig_df['UnitPrice'].fillna(0).astype(np.int)

# 3.價格處理
orig_df['totalprice'] = orig_df['totalprice'].str.replace("總價", "")
orig_df['totalprice'] = orig_df['totalprice'].str.replace("萬/套", "")
orig_df['totalprice'] = orig_df['totalprice'].fillna(0).astype(np.int)

# 4.總價計算
for idx, row in orig_df.iterrows():
    if orig_df.loc[idx, 'totalprice'] == 0:
        orig_df.loc[idx, 'totalprice'] = (orig_df.loc[idx, 'aera'] * orig_df.loc[idx, 'UnitPrice']) // 10000
    if orig_df.loc[idx, 'UnitPrice'] != 0:
        orig_df.loc[idx, 'UnitPrice'] = '%.4f' % (orig_df.loc[idx, 'UnitPrice'] / 10000)
    elif orig_df.loc[idx, 'UnitPrice'] == 0:
        orig_df.loc[idx, 'UnitPrice'] = '%.4f' % (orig_df.loc[idx, 'totalprice'] / orig_df.loc[idx, 'aera'])
    # 將填補的aera為空處復原

# 5.面積復原,將填充的0去掉
orig_df['aera'] = orig_df['aera'].astype(np.str)
for idx, row in orig_df.iterrows():
    if orig_df.loc[idx, 'aera'] == '0':
        orig_df.loc[idx, 'aera'] = ''

# 6.總價
# 最大值
print("總價：")
imaxpos = orig_df['totalprice'].idxmax()
print("最貴房屋", orig_df.loc[imaxpos, "totalprice"], orig_df.loc[imaxpos, "name"])
# 最小值
iminpos = orig_df['totalprice'].idxmin()
print("最便宜房屋", orig_df.loc[iminpos, "totalprice"], orig_df.loc[iminpos, "name"])
# 中位數
print("中位數", orig_df['totalprice'].median())

# 7.單價
# 最大值
print("單價：")
idmaxpos = orig_df['UnitPrice'].astype(float).idxmax()
print("最貴房屋", orig_df.loc[idmaxpos, "UnitPrice"], orig_df.loc[idmaxpos, "name"])
# 最小值
idminpos = orig_df['UnitPrice'].astype(float).idxmin()
print("最便宜房屋", orig_df.loc[idminpos, "UnitPrice"], orig_df.loc[idminpos, "name"])
# 中位數
print("中位數", orig_df['UnitPrice'].median())

orig_df.to_csv("NewMydata.csv", header=True, encoding="gbk", mode='w+', index=False)

處理結果