前言

斗魚直播—每個人的直播平臺

閑著沒事兒，看起斗魚的游戲直播了，感覺挺有意思，就想著看看目前有多少人在直播，獲取直播的相關資訊存入csv中，想看什么一覽無余，

1. 獲取資料

斗魚直播間 https://www.douyu.com/directory/all

def __init__(self):
    self.url = 'https://www.douyu.com/directory/all'
    self.driver = webdriver.Chrome()
	self.driver.get(self.url)

2. 決議資料

self.driver.find_elements_by_xpath('//*[@id="listAll"]/section[2]/div[2]/ul/li/div')
獲取所有的直播間資訊的xpath，使用chrome driver自動化獲取，

def parse_data(self):
    room_list = self.driver.find_elements_by_xpath('//*[@id="listAll"]/section[2]/div[2]/ul/li/div')
    print(len(room_list))
    # 遍歷房間串列，從每一個房間中獲取資料
    data_list = []
    for room in room_list:
        temp = {}
        temp['title'] = room.find_element_by_xpath('./a[1]/div[2]/div[1]/h3').text
        temp['category'] = room.find_element_by_xpath('./a[1]/div[2]/div[1]/span').text
        temp['owner'] = room.find_element_by_xpath('./a[1]/div[2]/div[2]/h2/div').text
        temp['num'] = room.find_element_by_xpath('./a[1]/div[2]/div[2]/span').text
        temp['cover_link'] = room.find_element_by_xpath('./a[1]/div[1]/div[1]/img').get_attribute('src')

        data_list.append(temp)

    return data_list

3. 自動翻頁

while True:
     time.sleep(3)
     # parse
     data_list = self.parse_data()

     # save
     self.save_data(data_list)

     # next
     try:
         # el_next = self.driver.find_elements_by_xpath('//*[@class=" dy-Pagination-next"]')
         el_next = self.driver.find_element_by_xpath('//*[contains(text(), "下一頁")]')
         # 下拉滾動最下面，才能點擊下一頁按鈕
         self.driver.execute_script('scrollTo(0, 100000)')
         el_next.click()

         self.num += 120

         if self.num > 200:
             break

     except:
         break

4. 保存資料

import csv

 # self.driver.implicitly_wait(10)
 self.csv_file = open('douyuLive.csv', 'w', encoding='utf-8')
 # 獲取csv的writer物件
 self.writer = csv.writer(self.csv_file)
 # 初始化表頭
 self.writer.writerow(['title', 'category', 'owner', 'num', 'cover_link'])
 
def save_data(self, data_list):
    for data in data_list:
        self.writer.writerow(data.values())

5. 完整代碼

from selenium import webdriver
import time
import csv

class Douyu(object):

    def __init__(self):
        self.url = 'https://www.douyu.com/directory/all'
        self.driver = webdriver.Chrome()
        # self.driver.implicitly_wait(10)
        self.csv_file = open('douyuLive.csv', 'w', encoding='utf-8')
        # 獲取csv的writer物件
        self.writer = csv.writer(self.csv_file)
        # 初始化表頭
        self.writer.writerow(['title', 'category', 'owner', 'num', 'cover_link'])
        # 多少條資料
        self.num = 0

    def parse_data(self):
        room_list = self.driver.find_elements_by_xpath('//*[@id="listAll"]/section[2]/div[2]/ul/li/div')
        print(len(room_list))
        # 遍歷房間串列，從每一個房間中獲取資料
        data_list = []
        for room in room_list:
            temp = {}
            temp['title'] = room.find_element_by_xpath('./a[1]/div[2]/div[1]/h3').text
            temp['category'] = room.find_element_by_xpath('./a[1]/div[2]/div[1]/span').text
            temp['owner'] = room.find_element_by_xpath('./a[1]/div[2]/div[2]/h2/div').text
            temp['num'] = room.find_element_by_xpath('./a[1]/div[2]/div[2]/span').text
            temp['cover_link'] = room.find_element_by_xpath('./a[1]/div[1]/div[1]/img').get_attribute('src')

            data_list.append(temp)

        return data_list

    def save_data(self, data_list):
        for data in data_list:
            self.writer.writerow(data.values())

    def run(self):
        # url
        # driver
        # get
        self.driver.get(self.url)

        while True:
            time.sleep(3)
            # parse
            data_list = self.parse_data()

            # save
            self.save_data(data_list)

            # next
            try:
                # el_next = self.driver.find_elements_by_xpath('//*[@class=" dy-Pagination-next"]')
                el_next = self.driver.find_element_by_xpath('//*[contains(text(), "下一頁")]')
                # 下拉滾動最下面
                self.driver.execute_script('scrollTo(0, 100000)')
                el_next.click()

                self.num += 120
				# 測驗代碼
                # if self.num > 200:
                #    break

            except:
                break


if __name__ == '__main__':
    # 實體化物件
    douyu = Douyu()
    # 開啟爬蟲主程式
    douyu.run()
    # 關閉檔案流
    douyu.csv_file.close()
    print(f'共有{douyu.num}條直播資料，下載完畢！')
    # 殺死行程
    douyu.driver.quit()

6. 效果展示

加油!

感謝!

努力!