python 獲取最新房價資訊-以北京房價為例-有解無憂

整個資料獲取的資訊是通過房源平臺獲取的，通過下載網頁元素并進行資料提取分析完成整個程序，

【閱讀全文】

file

匯入相關的網頁下載、資料決議、資料處理庫

from fake_useragent import UserAgent  # 身份資訊生成庫

from bs4 import BeautifulSoup  # 網頁元素決議庫
import numpy as np  # 科學計算庫
import requests  # 網頁下載庫
from requests.exceptions import RequestException  # 網路請求例外庫
import pandas as pd  # 資料處理庫

然后，在開始之前初始化一個身份資訊生成的物件，用于后面隨機生成網頁下載時的身份資訊，

user_agent = UserAgent()

撰寫一個網頁下載函式get_html_txt，從相應的url地址下載網頁的html文本，

def get_html_txt(url, page_index):
    '''
    獲取網頁html文本資訊
    :param url: 爬取地址
    :param page_index:當前頁數
    :return:
    '''
    try:
        headers = {
            'user-agent': user_agent.random
        }
        response = requests.request("GET", url, headers=headers, timeout=10)
        html_txt = response.text
        return html_txt
    except RequestException as e:
        print('獲取第{0}頁網頁元素失敗！'.format(page_index))
        return ''

撰寫網頁元素處理函式catch_html_data，用于決議網頁元素，并將決議后的資料元素保存到csv檔案中，

def catch_html_data(url, page_index):
    '''
    處理網頁元素資料
    :param url: 爬蟲地址
    :param page_index:
    :return:
    '''

    # 下載網頁元素
    html_txt = str(get_html_txt(url, page_index))

    if html_txt.strip() != '':

        # 初始化網頁元素物件
        beautifulSoup = BeautifulSoup(html_txt, 'lxml')

        # 決議房源串列
        h_list = beautifulSoup.select('.resblock-list-wrapper li')

        # 遍歷當前房源的詳細資訊
        for n in range(len(h_list)):
            h_detail = h_list[n]

            # 提取房源名稱
            h_detail_name = h_detail.select('.resblock-name a.name')
            h_detail_name = [m.get_text() for m in h_detail_name]
            h_detail_name = ' '.join(map(str, h_detail_name))

            # 提取房源型別
            h_detail_type = h_detail.select('.resblock-name span.resblock-type')
            h_detail_type = [m.get_text() for m in h_detail_type]
            h_detail_type = ' '.join(map(str, h_detail_type))

            # 提取房源銷售狀態
            h_detail_status = h_detail.select('.resblock-name span.sale-status')
            h_detail_status = [m.get_text() for m in h_detail_status]
            h_detail_status = ' '.join(map(str, h_detail_status))

            # 提取房源單價資訊
            h_detail_price = h_detail.select('.resblock-price .main-price .number')
            h_detail_price = [m.get_text() for m in h_detail_price]
            h_detail_price = ' '.join(map(str, h_detail_price))

            # 提取房源總價資訊
            h_detail_total_price = h_detail.select('.resblock-price .second')
            h_detail_total_price = [m.get_text() for m in h_detail_total_price]
            h_detail_total_price = ' '.join(map(str, h_detail_total_price))

            h_info = [h_detail_name, h_detail_type, h_detail_status, h_detail_price, h_detail_total_price]
            h_info = np.array(h_info)
            h_info = h_info.reshape(-1, 5)
            h_info = pd.DataFrame(h_info, columns=['房源名稱', '房源型別', '房源狀態', '房源均價', '房源總價'])
            h_info.to_csv('北京房源資訊.csv', mode='a+', index=False, header=False)

        print('第{0}頁房源資訊資料存盤成功！'.format(page_index))
    else:
        print('網頁元素決議失敗！')

撰寫多執行緒處理函式，初始化網路網頁下載地址，并使用多執行緒啟動呼叫業務處理函式catch_html_data，啟動執行緒完成整個業務流程，

import threading  # 匯入執行緒處理模塊


def thread_catch():
    '''
    執行緒處理函式
    :return:
    '''
    for num in range(1, 50, 3):
        url_pre = "https://bj.fang.lianjia.com/loupan/pg{0}/".format(str(num))
        url_cur = "https://bj.fang.lianjia.com/loupan/pg{0}/".format(str(num + 1))
        url_aft = "https://bj.fang.lianjia.com/loupan/pg{0}/".format(str(num + 2))

        thread_pre = threading.Thread(target=catch_html_data, args=(url_pre, num))
        thread_cur = threading.Thread(target=catch_html_data, args=(url_cur, num + 1))
        thread_aft = threading.Thread(target=catch_html_data, args=(url_aft, num + 2))
        thread_pre.start()
        thread_cur.start()
        thread_aft.start()


thread_catch()

資料存盤結果展示效果

file