01爬取當當網500本五星好評書籍

# import requests,re,json
# # 定義一個函式用來請求當當網的網頁資訊
# def request_dangdang(url):
#     try:
#         # 使用get請求
#         response = requests.get(url)
#         # 判斷回傳的狀態碼是否為200
#         if response.status_code == 200:
#             # 回傳接受的文本
#             return response.text
#     # 若訪問出現錯誤，就回傳空
#     except requests.RequestException:
#         return None
# # 定義函式用來決議訪問當當網收到的文本檔案
# def parse_text(html):
#     # 將正則運算式匹配符封裝，以便多次使用
#     pattern = re.compile(
#         # 用正則運算式決議出網頁上我們需要的書本名稱資訊
#         '<li>.*?list_num.*?(\d+).</div>.*?<img src="https://www.cnblogs.com/cong12586/p/(.*?)".*?.*?title="(.*?)">.*?>.*?>(.*?)</span>.*?>.*?target="_blank">(.*?)</a>.*?>.*?<span>(.*?)</span></div>.*?<p><span\s>￥(.*?)</span>.*?</li>',
#         re.S
#     )
#     # 找出所有的書本資訊
#     results = re.findall(pattern,html)
#     for result in results:
#         # 用生成器的方式生成資料
#         yield{
#             "range":results[0],
#             "image":results[1],
#             "title":results[2],
#             "recommend":results[3],
#             "author":results[4],
#             "times":results[5],
#             "price":results[6]
#         }
# # 將決議到的資料寫入檔案中
# def write_to_file(result):
#     print("準備開始，寫入資料  ====>" + str(result))
#     with open("book.txt",'a',encoding = "utf-8") as f:
#         f.write(json.dumps(result,ensure_ascii=False) + '\n')
# # 主函式
# def main(page):
#     url = 'http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-' + str(page)
#     # 獲取當當網的頁面
#     html = request_dangdang(url)
#     print("獲取網頁成功")
#     # 決議出我們想要的資訊
#     results = parse_text(html)
#     print("決議資訊成功")
#     # 然后將資訊依次寫入
#     for result in results:
#         write_to_file(result)
#     print("寫入資訊成功")
#
# if __name__ == "__main__":
#     for index in range(1,26):
#         print(index)
#         main(index)
import requests
import re
import json


def request_dandan(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
    except requests.RequestException:
        return None


def parse_result(html):
    print("正則運算式")
    pattern = re.compile(
        # '<li>.*?list_num.*?(\d+).</div>.*?<img src="https://www.cnblogs.com/cong12586/p/(.*?)".*?.*?title="(.*?)">.*?>.*?>(.*?)</span>.*?>.*?target="_blank">(.*?)</a>.*?>.*?<span>(.*?)</span></div>.*?<p><span>\s>￥(.*?)</span>.*?</li>',
        '<li>.*?list_num.*?(\d+).</div>.*?<img src="https://www.cnblogs.com/cong12586/p/(.*?)".*?.*?title="(.*?)">.*?>.*?>(.*?)</span>.*?>.*?target="_blank">(.*?)</a>.*?>.*?<span>(.*?)</span></div>.*?<p><span >.yen;(.*?)</span>.*?</li>',
        re.S)
    items = re.findall(pattern, html)
    for item in items:
        yield {
            'range': item[0],
            'iamge': item[1],
            'title': item[2],
            'recommend': item[3],
            'author': item[4],
            'times': item[5],
            'price': item[6]
        }


def write_item_to_file(item):
    print('開始寫入資料 ====> ' + str(item))
    with open('book.txt', 'a', encoding='UTF-8') as f:
        f.write(json.dumps(item, ensure_ascii=False) + '\n')


def main(page):
    url = 'http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-' + str(page)
    html = request_dandan(url)
    # print(html)
    # print("請求網頁成功")
    items = parse_result(html)  # 決議過濾我們想要的資訊
    # print("決議網頁成功")
    for item in items:
        # print("開始寫入資料")
        write_item_to_file(item)


if __name__ == "__main__":
    # 回圈爬取26頁
    for i in range(1,26):
        main(i)

標籤：Python

上一篇：當你忘記網站上的密碼時怎么辦？Python如何快速幫你找回？

下一篇：獲取標題和內容