python爬取pdf是空檔案

# !/usr/bin/env python

# -*- coding: utf-8 -*-

"""

作者:xiaofeng

日期:25/5/2021  8:23 上午

Python版本：3.7.4

"""

import sys

sys.path.append("./")

import time

import requests

from bs4 import BeautifulSoup

import urllib3

urllib3.disable_warnings()

from utils.mysql_database import MySQL





cookies = {'ASP.NET_SessionId': 'ltwutnzgoksolnh5plvbgom5', '__jsluid_h': '0dcd428869d5ddfa9f93734877768ebd', 'ADHOC_MEMBERSHIP_CLIENT_ID1.0': '1bdb8472-8581-4d57-1f15-6886e481c94e', 'hxck_cd_sourceteacher': 'sR/uPcnSSZVIdShwHag3RAnrY9aauRbMjEnRBtq/NF1ooDP7obDVPgaQGWxsj76JKxyO3FI4Vc44fDgs6a1RPPpEobT+rbs69gkUhZXkQRoriNmMhXYGi/Kg9Ge0W8OrEI7l3vb8+Lqxxc3S7ShzRhKqh6bc78GcuOlpZb32nz0skyJDPkyAQqaKNc5MYiP97iqqVoya6Js', 'hxck_cd_channel_order_mark1': '4001000001_auto', 'hxck_cd_channel': 'tKK6EMkJ7JK75WOJ/qluxbbMrhZQZtn9if6/Tggkwv0C9bkEdyIEtGS3AuATPaYjdIt/JM0butmcm0ugDRAOQLZ9K84yt3NDkdLWbt3B6UKb6vH2CyPMpRzCakALWJ0ET22e0EDh5Fg', 'HexunTrack': 'SID', 'UM_distinctid': '179a0ef15f934a-095394371ec4c1-37607201-1fa400-179a0ef15faba8', 'CNZZDATA1261865322': '1800598666-1621899014-|1621899014', 'Hm_lvt_cb1b8b99a89c43761f616e8565c9107f': '1621902694', 'hxck_fsd_lcksso': '765785F554A89C312D8941DA89784E8DA6F07A4DD0C8A7A01756B75532CBC92A9AF61DA2A10903428FD9C7FC05571766C495F9C6E6533A8E8A3774708CE623A80C4187A3EF72C130918420129BC6A79A375822E3B617E696711F30060E06725F0C072AC7F0B58AD6C801EDC29EFE6355DD132D6C15AD991C443F101A5098B016DB8260A7CC55FD1BA2761CB14AE42A503C7B409ADA0E30796F444D8FDFCAA827CBE7B840C02B95B605E8BCA36760606A', 'appToken': 'pc,other,chrome,hxAppSignId69134280520491461621903400822,PCDUAN', 'Hm_lvt_81ff19c9eb1c05cdfeacb05d2036f066': '1621903401', 'Hm_lpvt_81ff19c9eb1c05cdfeacb05d2036f066': '1621903401', 'cn_1263247791_dplus': '{"distinct_id": "179a0ef15f934a-095394371ec4c1-37607201-1fa400-179a0ef15faba8","userFirstDate": "20210525","userID": "32292139","userName": "wxmi92r4bx","userType": "loginuser","userLoginDate": "20210525","$_sessionid": 0,"$_sessionTime": 1621903815,"$dp": 0,"$_sessionPVTime": 1621903815}', 'userToken': '32292139|0000|0,dG5BpY0afu0XRdGr7cu5AGxVNVoAoU8uZn4D7HR9oE1VDxJlPLV8ZpSB3QtLylj5AuQOUmUm8SVmFm4j4T042k/BD9a/HRLPKNbXRpWJJ8I7ycZWKhCuPuwxEPmr+YMNy1T9WDYekIcDRnZyvQOGmOv2hwOJo8+4xJawWZl5+PdmIUHsA2NwuPahwH9JbfSEB7XcDuZCTttz/qQTmpZLUQ', 'hxck_sq_common': 'SnapCookie', 'Hm_lpvt_cb1b8b99a89c43761f616e8565c9107f': '1621903840'}





mysql = MySQL("test_db")

sql_list = list()





def down_load_pdf(url, name, headers_pdf):

    data = url.split("?")[1]

    post_data = dict()

    verifycode = data.split("&")[0].split("=")[1]

    fn = data.split("&")[1][3:]

    post_data["verifycode"] = verifycode

    post_data["fn"] = fn

    response = requests.get(url=url, headers=headers_pdf, data=https://bbs.csdn.net/topics/post_data, cookies=cookies)

    with open("download_pdf/company_research/{}.pdf".format(name), "wb") as f:

        f.write(response.content)





def main():

    url_info = ["http://yanbao.stock.hexun.com/dzgg1022315.shtml",

                "http://yanbao.stock.hexun.com/dzgg1022288.shtml",

                "http://yanbao.stock.hexun.com/dzgg1022286.shtml",

                "http://yanbao.stock.hexun.com/dzgg1022276.shtml",

                "http://yanbao.stock.hexun.com/dzgg1022267.shtml",

                "http://yanbao.stock.hexun.com/dzgg1023589.shtml"]

    for url in url_info:

        url_page = 1

        source_page = "http://yanbao.stock.hexun.com/listnews1_" + str(url_page) + ".shtml"

        headers_pdf = {

            "Host": "yanbao.stock.hexun.com",

            "Referer": source_page,

            "Upgrade-Insecure-Requests": "1",

            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"

        }

        pdf_name = url.replace("http://yanbao.stock.hexun.com/", "").replace(".shtml", "")

        response = requests.get(url=url, headers=headers_pdf, cookies=cookies)

        pdf_html = response.content.decode("gbk")

        soup_ = BeautifulSoup(pdf_html, "html.parser")

        for p_tag in soup_.find_all('a', class_='check-pdf'):

            pdf_url = p_tag.attrs['href'] #獲取href屬性,抓取pdf的url資訊

            if ("verifycode" and "fn") in pdf_url:

                print(pdf_url)

                down_load_pdf(pdf_url, pdf_name, headers_pdf)

            else:

                print("pdf的url獲取失敗: " + pdf_url)

        time.sleep(3)





if __name__ == "__main__":

    main()

uj5u.com熱心網友回復：

import requests

from lxml import etree





Cookie = 'UM_distinctid=17887a54f0b10d-06b524a5fabe19-5771031-1fa400-17887a54f0c1ea; hxck_cd_sourceteacher=sR%2fuPcnSSZVIdShwHag3RAnrY9aauRbMjEnRBtq%2fNF1ooDP7obDVPgaQGWxsj76JKxyO3FI4Vc6fQ3EdP48WO2CqFkaCVZFZDfun86yj7x6OKsTsbHgxOYWFQntuUdH8e28Fw9aiqM0XlYwcgGGgZyIuxTrCCF1o0OblSiXH353cG3SfHYeD5n%2fLOCyVmzuR580ywnqrbhUCeRYKfuRltRzNTs8Fwhw%2fEFl08cEvhkupzucYCP3EK%2ffvIRrdc8on2MnygOEBUpKOSj%2fGKP5MIGtMcR9pUrL61bNASZ96Idgf4Hl8YeNSmxkfsVGs6%2b4Sd6PwBBxaNxfsFhGO53B33w%3d%3d; hxck_cd_channel=tKK6EMkJ7JK75WOJ%2fqluxbbMrhZQZtn9if6%2fTggkwv3skKxI8jKUM1LEJMFTYZazf6xG4ZAsYVrPow6nQBjVDLHsdcQTA7BH5mFxk1pTrn4YltSwneAHqW7q0KrjTwlSJJVl4Y1nSkvuLvi3zPIiG2mvdCcNOXTJdXTsKmf50R%2bk6sXj5RRxUDyQxDKqssnqMKP29RLq3oUJCpTKHoSTJai8LKGBjA3jcKdYynaeoinAzaUgbvZYPc0hcKC1VtUXs0Nf88feKDz1VkkDSNleoqBY84g60DI8; appToken=pc%2Cother%2Cchrome%2ChxAppSignId2179467776209735.81617263076724%2CHXGG20190415; ASP.NET_SessionId=2pwsjst2xo02cpldezobwbiz; __jsluid_h=193d56abe14cd16c207d8d99a03b74f4; ADHOC_MEMBERSHIP_CLIENT_ID1.0=a2f8294f-2779-2fa9-0c0a-d48f8149442b; hxck_cd_channel_order_mark1=4001000001_auto; HexunTrack=SID=202103311737410742b2784f36e644b85bf8fcc5eb0ea4d9b&CITY=36&TOWN=0; Hm_lvt_cb1b8b99a89c43761f616e8565c9107f=1621929913; appToken=pc%2Cother%2Cchrome%2ChxAppSignId3095319029421017.51621930176027%2CPCDUAN; Hm_lvt_81ff19c9eb1c05cdfeacb05d2036f066=1621930176; Hm_lpvt_81ff19c9eb1c05cdfeacb05d2036f066=1621930176; __utma=194262068.525988224.1617183468.1617183468.1621930193.2; __utmc=194262068; __utmz=194262068.1621930193.2.2.utmcsr=yanbao.stock.hexun.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmb=194262068.1.10.1621930193; cn_1263247791_dplus=%7B%22distinct_id%22%3A%20%2217887a54f0b10d-06b524a5fabe19-5771031-1fa400-17887a54f0c1ea%22%2C%22userFirstDate%22%3A%20%2220210331%22%2C%22userID%22%3A%20%220%22%2C%22userName%22%3A%20%22%22%2C%22userType%22%3A%20%22loginuser%22%2C%22userLoginDate%22%3A%20%2220210331%22%2C%22%24_sessionid%22%3A%200%2C%22%24_sessionTime%22%3A%201621930200%2C%22%24dp%22%3A%200%2C%22%24_sessionPVTime%22%3A%201621930200%2C%22initial_view_time%22%3A%20%221617183416%22%2C%22initial_referrer%22%3A%20%22http%3A%2F%2Ffunds.hexun.com%2Fhotnews%2F%22%2C%22initial_referrer_domain%22%3A%20%22funds.hexun.com%22%7D; hxck_fsd_lcksso=765785F554A89C310D147ABE887DEFB686F4ABB3B9C4FBF8EE9C56240E980CC3EA75D4B18E7F38AA24EBDB2DBCE54469BF29F585974E9E7FC59ECF6FAA019AA363F43B11D4589B6131B6D8BA5D8CC80CE4B58FFAB159EB16748E6F43D49C8CAFA972F61838B555179807A614D2CC42A3B94DC13E6A26D2E5A1BFB278447703B4FADA5A61E8713BAAD480E16CC484AA973A85E53F0439E6715B6F6082087C280B2E5DB74EEC8072FBF8A3A6440AA0075C; hxck_sq_common=SnapCookie=5G5BS6nFgypK79UvNxXgzWcOMkeKNdmHxgtDwowHtUe%2fkDcupxwowejTXPAUdVnp6EL8yi%2fp%2b04ZA07WEXVU8lfsHSrGeYSAm47WVRCM6xcxl4hRubH%2bxQ%3d%3d; userToken=32294024%7c0000%7c0%2c1Vm65shj7bMXMpQ%2fam3Dp84RMmA%2fsYnnaQjNVgpOUJXmbQZuiNl1DFeZ5OqSuLu%2fo3h0KqKZGASzwyABueIDNTDw0gV3CNzUQbQDWpc%2fYazXAz4OTz33NKu4%2fovu61Y5JP%2be%2fYbOgSMVfOXVMusH5DyPXld3E5J9vGBaog%2f7niv%2bUB1%2fmF9PngAKjIWYmTS%2ft8KAL8npoSc3edhTQjakfA%3d%3d; CNZZDATA1261865322=1565002879-1621926022-%7C1621931424; Hm_lpvt_cb1b8b99a89c43761f616e8565c9107f=1621931507'





def get_pdf_url():

    url = 'http://yanbao.stock.hexun.com/dzgg1022267.shtml'

    headers = {

        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',

        'Accept-Encoding': 'gzip, deflate',

        'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',

        'Connection': 'keep-alive',

        'Cookie': Cookie,

        'Referer': 'http://yanbao.stock.hexun.com/listnews1_1.shtml',

        'Host': 'yanbao.stock.hexun.com',

        'Upgrade-Insecure-Requests': '1',

        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36'

    }

    response = requests.get(url=url, headers=headers)

    tree = etree.HTML(response.text)

    pdf_url = tree.xpath('//a[@class="check-pdf"]/@href')[0]

    pdf_name = url.replace("http://yanbao.stock.hexun.com/", "").replace(".shtml", "")

    print(pdf_url, pdf_name)

    return pdf_url, pdf_name





def download_pdf(url, pdf_name):

    headers = {

        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',

        'Accept-Encoding': 'gzip, deflate',

        'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',

        'Connection': 'keep-alive',

        'Cookie': Cookie,

        'Host': 'bbs.r.hexun.com',

        'Referer': 'http://yanbao.stock.hexun.com/',

        'Upgrade-Insecure-Requests': '1',

        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36'

    }

    response = requests.get(url=url, headers=headers)

    with open('{}.pdf'.format(pdf_name), 'wb') as f:

        f.write(response.content)





if __name__ == "__main__":

    pdf_url, pdf_name = get_pdf_url()

    download_pdf(pdf_url, pdf_name)

標籤：腳本語言(Perl/Python)

上一篇：用pandas讀取mysql資料時FROM_UNIXTIME格式化日期不正確

下一篇：lingo混合整數線性規劃編程