網路爬蟲實體代碼優化 面試重點

我在“python3開發桌面軟體之程式主體”一文中已經分享了919網監助手1.0程式的主體源代碼（即events.py中的代碼），后用迭代器、生成器運算式等對代碼進行優化，使程式運行更快、更穩定，優化后的代碼如下：

#coding:utf-8
import wx
import configparser
import requests
import re
import os
import string
from bs4 import BeautifulSoup
from openpyxl import load_workbook

class Getlike():
def __init__(self, filePath,row1,erow1,bcol1,obj1):
self.filePath=filePath
self.row1=row1
self.erow1=erow1
self.bcol1=bcol1
self.obj1=obj1
def main1(self):
list1 = self.rexcel()
hr1 = []
st1 = []
for site in list1:
if site == None:
break

if site[0:4] == 'http':
demo = self.get_html0(site)

if demo == '':
hr1.append(site)
st1.append(u'獲取主頁資料失敗')
continue

soup = BeautifulSoup(demo, "html.parser")
self.obj1.SetLabel(site)
if site[-1] == '/':
site = site[0:-1]
i = 0
for a in soup.find_all('a'):
i += 1
if a.has_attr('href'):
if (a['href'][0:4] != 'http') and ('@' not in a['href']) and (
'javascript:' not in a['href']) and ('javasrcipt:' not in a['href']) and (
'tel:' not in a['href']): # 非站外鏈接且非正常鏈接
urlg = a['href']
if len(urlg) > 0:
if urlg[0] != '/':
urlg = '/' + urlg
urlhb = site + urlg
if urlhb in hr1: # 過慮重復鏈接
continue
hr1.append(urlhb)
st1.append(a.string)
if i == 0:
hr1.append(site)
st1.append(u'該網頁無鏈接')

hr2=iter(hr1)# 創建迭代器物件
st2=iter(st1)# 創建迭代器物件
self.wexcel0(hr2, st2)
def get_html0(self,url):
headers = {
'User-Agent': 'Mozilla/5.0(Windows NT 10.0; WOW64; Trident/7.0; rv:11.0)\
like Gecko'
} # 模擬瀏覽器訪問
try:
r = requests.get(url, timeout=30, headers=headers) # 請求訪問網站
r.raise_for_status()
r.encoding = r.apparent_encoding

if r.status_code == 200:
return r.text
else:
return ''
except:
return ''

def rexcel(self): # 讀excel檔案
list1 = []
if not os.path.exists(self.filePath):
wx.MessageBox(u'沒找到excel檔案', u'提示')
return list1
wb = load_workbook(self.filePath,read_only=True)
#wb = load_workbook(self.filePath)
ws = wb.active
j =self.row1
if self.erow1<j :
erow2=ws.max_row
else:
erow2 =self.erow1
list1=(ws.cell(row=r, column=self.bcol1).value for r in range(j,erow2)) #生成器運算式
return list1

def wexcel0(self, hr1, st1): # 寫excel檔案，站內鏈接
if not os.path.exists(self.filePath):
return 0
wb = load_workbook(self.filePath)
ws = wb.active
nrows = ws.max_row+2
if ws.column_dimensions['B'].width<10:
upper_string = string.ascii_uppercase[:5]
for co in upper_string:
ws.column_dimensions[co].width = 50 # 設定列寬

for r in range(1, nrows): #避免覆寫原資料
if ws.cell(row=r, column=self.bcol1+1).value is None:
break

for (x,y) in zip(hr1,st1):
ws.cell(row=r, column=self.bcol1 + 1).value = https://www.cnblogs.com/shabge/p/x
ws.cell(row=r, column=self.bcol1 + 2).value =https://www.cnblogs.com/shabge/p/y
r = r + 1
if r> 1048575:
break
try:
wb.save(self.filePath)
wb.close()
except:
wx.MessageBox(u'權限被拒絕，請關閉打開的excel檔案', u'提示')
return 0
return 1

class Mgzfind(Getlike):
def __init__(self, filePath, row1, erow1, bcol1, obj1,mgz):
Getlike.__init__(self, filePath, row1, erow1, bcol1, obj1)
self.mgz=mgz

def main2(self):
list1= Getlike.rexcel(self)
regex1 = self.mgz # 敏感詞之間用|分隔
pattern = \
r'([\u4e00-\u9fa5]{1,}(' + regex1 + '))'
re_compile = re.compile(pattern)
list11=[]
list2 = []
for m in list1:
list11.append(m)
content = ''
#s1 = list1[m]
if m == None:
break

if m[0:4] == 'http':
webcontent = self.get_html(m)
else:
continue

if webcontent == '':
list2.append(u'獲取頁面資料失敗')
continue
else:
for n in re_compile.finditer(webcontent):
content = content + n.group() + ';'
if content=='':
content ='nothing'
list2.append(content)
self.obj1.SetLabel(m)
list1s=iter(list11) # 創建迭代器物件
list2s = iter(list2) # 創建迭代器物件
self.wexcel(list1s,list2s)

def get_html(self,url):
headers = {
'User-Agent': 'Mozilla/5.0(Windows NT 10.0; WOW64; Trident/7.0; rv:11.0)\
like Gecko'
} # 模擬瀏覽器訪問
requests.adapters.DEFAULT_RETRIES = 3
# s = requests.session()
# s.keep_alive = False
try:
r = requests.get(url, timeout=10, headers=headers) # 請求訪問網站
r.raise_for_status()
r.encoding = r.apparent_encoding

if r.status_code == 200:
return r.text
else:
return ''
except:
return ''

def wexcel(self,list1, list2): # 寫excel檔案,敏感詞
if not os.path.exists(self.filePath):
return 0
wb = load_workbook(self.filePath)
ws = wb.active
col = 2 + self.bcol1
nrows = ws.max_row+2

for r in range(1, nrows): #避免覆寫原資料
if ws.cell(row=r, column=col+1).value is None:
break

for (x,y) in zip(list1,list2): #在空白處追加資料
ws.cell(row=r, column=col).hyperlink = x
ws.cell(row=r, column=col + 1).value =https://www.cnblogs.com/shabge/p/y
r = r + 1
if r > 1048575:
break
try:
wb.save(self.filePath)
wb.close()
except:
wx.MessageBox(u'權限被拒絕，請關閉打開的excel檔案', u'提示')
return 0
return 1

#主表單事件
class MyFrame1_events():
def __init__(self, frame):
self.frame = frame

#設定圖示
self.frame.SetIcon(wx.Icon(u't1.ico', wx.BITMAP_TYPE_ICO))
self.readcon()

#系結事件
self.frame.Bind(wx.EVT_CLOSE, self.MyFrame1_OnClose)
self.frame.m_radioBtn15.Bind(wx.EVT_RADIOBUTTON, self.radio5)
self.frame.m_radioBtn6.Bind(wx.EVT_RADIOBUTTON, self.radio6)
self.frame.m_button14.Bind(wx.EVT_BUTTON, self.openf)
self.frame.m_button12.Bind(wx.EVT_BUTTON, self.mymain)

# 自定義函式
def readcon(self):
cf = configparser.ConfigParser()
cf.read("config.ini",encoding="utf-8")
secs = cf.sections()
filePath = cf.get("excel", "path")
if self.frame.m_radioBtn15.GetValue():
brow1 = cf.get("excel", "brow1")
erow1 = cf.get("excel", "erow1")
bcol1 = cf.get("excel", "bcol1")
else:
brow1 = cf.get("excel", "brow2")
erow1 = cf.get("excel", "erow2")
bcol1 = cf.get("excel", "bcol2")

word = cf.get("keyword", "word")
word = word.replace("，", "|")
self.frame.m_textCtrl25.Clear()
#filePath=filePath.decode('gbk', 'ignore').encode('utf-8')
self.frame.m_textCtrl25.WriteText(filePath)
self.frame.m_spinCtrl1.SetValue(brow1)
self.frame.m_spinCtrl2.SetValue(erow1)
self.frame.m_spinCtrl3.SetValue(bcol1)
#word1=word.decode('gbk', 'ignore').encode('utf-8')
self.frame.m_textCtrl32.Clear()
self.frame.m_textCtrl32.WriteText(word)
return
def savecon(self):
filePath = self.frame.m_textCtrl25.GetValue()
#brow1 = self.frame.m_spinCtrl1.GetValue()
erow1 = self.frame.m_spinCtrl2.GetValue()
bcol1 = self.frame.m_spinCtrl3.GetValue()
mgz = self.frame.m_textCtrl32.GetValue()
mgz = mgz.replace("，", "|")
cf = configparser.ConfigParser()
cf.read("config.ini", encoding="utf-8")
secs = cf.sections()
cf.set("keyword", "word", mgz)
cf.set("excel", "path", filePath)
if self.frame.m_radioBtn15.GetValue():
cf.set("excel", "brow1",str(erow1))
cf.set("excel", "erow1", str(erow1+10))
self.frame.m_spinCtrl2.SetValue(erow1 + 10)
cf.set("excel", "bcol1", str(bcol1))
else:
cf.set("excel", "brow2", str(erow1))
cf.set("excel", "erow2", str(erow1+30))
self.frame.m_spinCtrl2.SetValue(erow1 + 30)
cf.set("excel", "bcol2", str(bcol1))
cf.write(open('config.ini', "r+",encoding='UTF-8'))
self.frame.m_spinCtrl1.SetValue(erow1)
return

#關閉視窗事件
def MyFrame1_OnClose(self, event):
"""表單關閉"""
if self.frame.m_button12.GetLabel()==u'確定':
frame = event.GetEventObject()
frame.Destroy()
else:
wx.MessageBox(u'程式進在運行，不能關閉視窗', u'提示')
frame = event.GetEventObject()
frame.Destroy()
event.Skip()

# 瀏覽excel檔案事件
def openf(self, event):
openFileDialog = wx.FileDialog(self.frame, "請選擇要打開的Excel檔案", "", "",
"Excel格式 (*.xlsx)|*.xlsx",
wx.FD_OPEN | wx.FD_FILE_MUST_EXIST)

if openFileDialog.ShowModal() == wx.ID_OK:
filePath = openFileDialog.GetPath()
self.frame.m_textCtrl25.Clear()
self.frame.m_textCtrl25.WriteText(filePath)
openFileDialog.Destroy()

event.Skip()

def radio5(self, event):
self.readcon()
event.Skip()
def radio6(self, event):
self.readcon()
event.Skip()

def mymain(self, event):
#self.frame.m_button12.Enable(False)
self.frame.m_button12.SetLabel('正在運行...')
filePath = self.frame.m_textCtrl25.GetValue()
row1 = self.frame.m_spinCtrl1.GetValue()
erow1=self.frame.m_spinCtrl2.GetValue()
bcol1=self.frame.m_spinCtrl3.GetValue()
obj1=self.frame.m_staticText11
obj1.SetLabel('正在讀取網址...')
mgz=self.frame.m_textCtrl32.GetValue()
mgz =mgz.replace("，","|")
if self.frame.m_radioBtn15.GetValue():
emp1 =Getlike(filePath,row1,erow1,bcol1,obj1)
emp1.main1()
else:
emp2 = Mgzfind(filePath, row1, erow1, bcol1, obj1,mgz)
emp2.main2()
self.savecon()
obj1.SetLabel('完成')
#self.frame.m_button12.Enable(True)
wx.MessageBox(u'完成', u'提示')
self.frame.m_button12.SetLabel('確定')
event.Skip()
注意：很多人學Python程序中會遇到各種煩惱問題解決不了，為此小編建了個Python全堆疊免費答疑交流.裙 ：留而思思思零期思無（數字的諧音）轉換下可以找到了，不懂的問題有老司機解決里面還有最新Python教程專案可拿,，一起相互監督共同進步！
本文的文字及圖片來源于網路加上自己的想法,僅供學習、交流使用,不具有任何商業用途,著作權歸原作者所有,如有問題請及時聯系我們以作處理，

標籤：Python

上一篇：使用 GitHub 開源專案申請 IntelliJ License

下一篇：Python - 開發截圖識別OCR小工具