爬蟲 1

1. 使用requests 發送GET、POST請求，

"""使用requests庫，發送請求"""
import requests
from fake_useragent import UserAgent


# GET 請求
def requests_get(headers):
    url = "https://www.baidu.com/s"
    params = {
        "wd": "快代理"
    }
    r = requests.get(url, headers=headers, params=params)
    print('---回應值為: ', r.status_code, '---編碼為: ', r.encoding)
    r.encoding = r.apparent_encoding
    print(r.request.headers)
    print(r.text)


# POST 請求
def requests_post(headers):
    url = "https://www.kuaidaili.com/login/"
    login_info = {
        "username": "...",
        "passwd": "..."
    }
    r = requests.post(url, headers=headers, data=https://www.cnblogs.com/leafchen/p/login_info)
    print('---回應值為: ', r.status_code, '---編碼為: ', r.encoding)
    print(r.text)

def main():
    
    headers = {
        'User-Agent': UserAgent().chrome
    }
    # 發送get請求
    requests_get(headers)
    
    # 發送post請求
    requests_post(headers)


if __name__ == '__main__':
    main()

簡單示例: 爬取唯美女生網站中某頁面資料

"""爬取 唯美女生 網站圖片"""
import requests
import re
import os
import time


# 1. 請求網頁
myheaders = {'User-Agent': 'Mozilla/5.0'}
# url = "http://pic.netbian.com"
url = 'http://pic.netbian.com/4kmeinv'
response = requests.get(url, headers=myheaders)

# 2. 處理回應資料， 正則匹配
html = response.text
img_urls = re.findall('<img src="https://www.cnblogs.com/leafchen/p/(.*?)" alt=".*?">', html)
print(img_urls)

# 3. 下載圖片
if not os.path.exists('彼岸圖片'):
    os.mkdir('彼岸圖片')
for img_url in img_urls:
    time.sleep(1)
    img_name = img_url.split('/')[-1]
    response = requests.get((url + img_url), headers=myheaders)
    with open('彼岸圖片/' + img_name, 'wb') as f:
        f.write(response.content)

2.  使用urllib 發送GET、POST請求， 當需要手動傳參時，可以使用urllib.parse中的：quote、urlencode來進行“中文”轉碼

"""使用 urllib 創建爬蟲"""
from urllib.request import urlopen
from urllib.request import Request      # 包裝爬蟲


url = 'https://www.baidu.com'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36'

}
# 創建Resquest物件，來包裝請求
request = Request(url, headers=headers, method='GET', data=https://www.cnblogs.com/leafchen/p/None)
# 發送請求
response = urlopen(request)
# 列印資訊
print('狀態碼：', response.getcode(), '真實請求地址：', response.geturl(), '狀態碼：', response.status, '請求頭：', response.getheaders())
info = response.read().decode()
print(info)

a. GET請求

"""GET 請求，當需要手動傳參，且引數為中文時,需要將引數轉碼
    單個引數時可以使用：quote,轉碼
    多個引數時：urlencode， 轉碼
"""
from urllib.request import urlopen, Request
from urllib.parse import quote

# 單個引數時可以使用：quote,轉碼
# url = "https://www.baidu.com/s?wd={}".format(quote("科技"))
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36"
}
# request = Request(url, headers=headers)
# response = urlopen(request, timeout=3)
# info = response.read().decode()
# print(info)


# 多個引數時：urlencode， 轉碼
from urllib.parse import urlencode

args = {
    'wd': "科技",
    'ie': 'utf-8'
    # ...
}

url = 'https://www.baidu.com/s?{}'.format(urlencode(args))
print(url)

request2 = Request(url, headers=headers)
response2 = urlopen(request2, timeout=3)
info2 = response2.read().decode()
print(info2)

b. POST請求， 在request物件創建時，封裝引數進去，

"""爬蟲， 簡單的post請求"""

from urllib.request import Request, urlopen
from urllib.parse import urlencode
from fake_useragent import UserAgent


url = "http://www.sxt.cn/index/login/login.html"

# 引數（此處為登錄的賬號和密碼）
form_data =https://www.cnblogs.com/leafchen/p/ {
    'user': '...',
    'password': '...'
}
f_data = urlencode(form_data)   # 轉碼，防止其中有中文而報錯

# 修改請求頭
headers = {
    "User-Agent": UserAgent().chrome
}

# 創建request物件，發送請求
try:
    request = Request(url, data=https://www.cnblogs.com/leafchen/p/f_data.encode(), headers=headers, )
    response = urlopen(request)
    info = response.read()
    # print(info.decode())
except Exception as err:
    print('err', err)

標籤：Python

上一篇：Python爬蟲實踐入門，超詳細

下一篇：執行檔案例外報錯：ImportError: attempted relative import with no known parent package