一、前言

疲憊的生活里總要有些溫柔夢想吧

目標URL：http://www.win4000.com/meinvtag4_1.html

爬取美桌網某個標簽下的美女壁紙，點進詳情頁可以發現，里面是一組套圖

二、網頁分析

翻頁查看 URL 變化規律：

http://www.win4000.com/meinvtag4_1.html
http://www.win4000.com/meinvtag4_2.html
http://www.win4000.com/meinvtag4_3.html
http://www.win4000.com/meinvtag4_4.html
http://www.win4000.com/meinvtag4_5.html

頁面里看到的每張圖片點擊進去有詳情頁，里面是套圖

詳情頁里套圖URL變化規律：

http://www.win4000.com/meinv216987_1.html
http://www.win4000.com/meinv216987_2.html
http://www.win4000.com/meinv216987_3.html

在網頁源代碼中也可以直接找到資料：

圖片名稱  下載鏈接
性感美女肌膚雪白撩人寫真圖片
http://pic1.win4000.com/pic/8/e0/24b989d57c.jpg

三、爬蟲基本思路

1. 獲取5頁的套圖的URL

def get_taotu_url():
    for i in range(1, 6):
        url = f'http://www.win4000.com/meinvtag4_{i}.html'
        headers = {
            'User-Agent': choice(user_agent)
        }
        # 發送請求  獲取回應
        rep = requests.get(url, headers=headers)
        # print(rep.status_code)    狀態碼  200
        # print(rep.text)
        html = etree.HTML(rep.text)
        taotu_url = html.xpath('//div[@class="tab_tj"]/div/div/ul/li/a/@href')
        # 過濾掉無效的url
        taotu_url = [item for item in taotu_url if len(item) == 39]
        # 一個頁面有24個圖片
        print(taotu_url, len(taotu_url), sep='\n')

2. 進入套圖詳情頁爬取圖片

def get_img(url):
    headers = {
        'User-Agent': choice(user_agent)
    }
    # 發送請求  獲取回應
    rep = requests.get(url, headers=headers)
    # 決議回應
    html = etree.HTML(rep.text)
    # 獲取套圖名稱   最大頁數
    name = html.xpath('//div[@class="ptitle"]/h1/text()')[0]
    os.mkdir(r'./女神套圖/{}'.format(name))
    max_page = html.xpath('//div[@class="ptitle"]/em/text()')
    # 字串替換  便于之后構造url請求
    url1 = url.replace('.html', '_{}.html')
    for i in range(1, int(max_page[0]) + 1):
        url2 = url1.format(i)
        sleep(randint(1, 3))
        reps = requests.get(url2, headers=headers)
        dom = etree.HTML(reps.text)
        src = dom.xpath('//div[@class="main-wrap"]/div[1]/a/img/@data-original')[0]
        file_name = name + f'第{i}張.jpg'
        img = requests.get(src, headers=headers).content
        with open(r'./女神套圖/{}/{}'.format(name, file_name), 'wb') as f:
            f.write(img)
            print(f'成功下載圖片：{file_name}')

3. 完整實作如下

# -*- coding: UTF-8 -*-
"""
@File    ：spider.py
@Author  ：葉庭云
@CSDN    ：https://yetingyun.blog.csdn.net/
"""
import requests
from random import choice, randint
from lxml import etree
import os
from concurrent.futures import ThreadPoolExecutor
from time import sleep


# 自己構造請求頭池  用于切換
user_agent = [
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
    ]

# 不存在檔案夾  就創建
if not os.path.exists('女神套圖'):
    os.mkdir('女神套圖')


# 獲取5頁的套圖的URL
def get_taotu_url():
    taotu_urls = []
    for i in range(1, 6):
        url = f'http://www.win4000.com/meinvtag4_{i}.html'
        headers = {
            'User-Agent': choice(user_agent)
        }
        # 發送請求  獲取回應
        rep = requests.get(url, headers=headers)
        # print(rep.status_code)    狀態碼  200
        # print(rep.text)
        html = etree.HTML(rep.text)
        taotu_url = html.xpath('//div[@class="tab_tj"]/div/div/ul/li/a/@href')
        # 過濾掉無效的url
        taotu_url = [item for item in taotu_url if len(item) == 39]
        # 一個頁面有24個圖片
        # print(taotu_url, len(taotu_url), sep='\n')
        taotu_urls.extend(taotu_url)

    return taotu_urls


# 進入套圖詳情頁爬取圖片
def get_img(url):
    headers = {
        'User-Agent': choice(user_agent)
    }
    # 發送請求  獲取回應
    rep = requests.get(url, headers=headers)
    # 決議回應
    html = etree.HTML(rep.text)
    # 獲取套圖名稱   最大頁數
    name = html.xpath('//div[@class="ptitle"]/h1/text()')[0]
    os.mkdir(r'./女神套圖/{}'.format(name))
    max_page = html.xpath('//div[@class="ptitle"]/em/text()')
    # 字串替換  便于之后構造url請求
    url1 = url.replace('.html', '_{}.html')
    # 翻頁爬取這組套圖的圖片
    for i in range(1, int(max_page[0]) + 1):
        # 構造url
        url2 = url1.format(i)
        # 休眠 
        sleep(randint(1, 3))
        # 發送請求  獲取回應
        reps = requests.get(url2, headers=headers)
        # 決議回應
        dom = etree.HTML(reps.text)
        # 定位提取圖片下載鏈接
        src = dom.xpath('//div[@class="main-wrap"]/div[1]/a/img/@data-original')[0]
        # 構造圖片保存的名稱
        file_name = name + f'第{i}張.jpg'
        # 請求下載圖片  保存圖片  輸出提示資訊
        img = requests.get(src, headers=headers).content
        with open(r'./女神套圖/{}/{}'.format(name, file_name), 'wb') as f:
            f.write(img)
            print(f'成功下載圖片：{file_name}')


# 主函式呼叫  開多執行緒
def main():
    taotu_urls = get_taotu_url()
    with ThreadPoolExecutor(max_workers=4) as exector:
        exector.map(get_img, taotu_urls)
    print('=================== 圖片全部下載成功啦！ =====================')
    
    
if __name__ == '__main__':
    main()