我正在嘗試生成代碼，該代碼將在紐約公共雇員工資這個網站上每年回圈，并單獨選擇每個可用的"Subagency/Type" 選項。挑戰在于選項的數量每年都在變化。我已經想到了幾種方法可以做到這一點，但一直無法弄清楚其中的任何一種。這些元素的 XPATH 遵循以下模式
所以在我意識到選項的數量每年都在變化之前，我一直在使用這樣的回圈：
for box in tqdm(range(1,74)):
link_to_use = '//*[@id="subagencyGroup"]/ul/li[' str(box) ']/span[1]'
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, link_to_use)))
driver.find_element(By.XPATH, link_to_use).click()
然后我將對結果執行一些代碼并通過取消單擊該元素來完成迭代，以便下一次迭代只會單擊下一個元素。我的想法是也許我可以使用 beautifulsoup 來獲取所有 li[N] 值并在頁面上找到它們的最大值 N，然后使用 that 1 作為我的范圍迭代器的上限值。
但后來我開始尋找更直接的途徑，即從頁面中獲取要單擊的元素串列，因此代碼看起來更像：
elem = driver.find_elements_by_xpath('//*[@id="subagencyGroup"]')
all_li = elem.find_elements_by_tag_name("li")
for el in tqdm(all_li):
WebDriverWait(driver, 10).until(EC.presence_of_element_located(el)
el.click()
但這似乎因某種我不知道的原因而停滯 - 盡管成功單擊了第一個選項。
完整的代碼如下，我將不勝感激任何指導。
# check if "page next" button is disabled
def check_for_break_condition(driver):
page_next_button = driver.find_element(By.ID, "data_loader")
styles = page_next_button.get_attribute("style")
if('none' in styles):
return True
else:
return False
# Loop Through Years
for yr in range(1, 15):
# Store URL to Use
url = 'https://www.seethroughny.net/payrolls'
# Initialize WebDriver
options = Options()
# options.headless = True
options.add_argument("start-maximized")
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
options.add_argument('--no-sandbox')
options.add_argument('--disable-application-cache')
options.add_argument('--disable-gpu')
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--window-size=1560,840")
driver = webdriver.Chrome('C:/Windows/System32/chromedriver.exe', options=options)
driver.get(url)
try:
driver.find_element(By.XPATH, '//*[@id="donate-banner"]/button/span').click()
except:
pass
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//*[@id="stny_payYear_2020"]/div')))
driver.find_element(By.CSS_SELECTOR, "#yearGroupHeading a").click()
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//*[@id="stny_payYear_2020"]/div')))
driver.find_element(By.XPATH, '//*[@id="stny_payYear_2020"]/div').click()
yr_path = '//*[@id="yearGroup"]/ul/li[' str(yr) ']'
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, yr_path)))
driver.find_element(By.XPATH, yr_path).click()
# Select Colleges
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//*[@id="subagencyGroupHeading"]/h4/a')))
driver.find_element(By.CSS_SELECTOR, "#subagencyGroupHeading a").click()
driver.find_element(By.CSS_SELECTOR, "#subagencyGroup .form-control").click()
# WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, '//*[@id="subagencyGroupHeading"]/h4/a')))
driver.find_element(By.XPATH, '//*[@id="subagencyGroup"]/div/input').send_keys("College")
time.sleep(5)
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, '//*[@id="subagencyGroup"]/ul/li[1]/span[1]')))
elem = driver.find_elements_by_xpath('//*[@id="subagencyGroup"]')
all_li = elem.find_elements_by_tag_name("li")
# Select SUNY and CUNY
for el in tqdm(all_li):
# link_to_use = '//*[@id="subagencyGroup"]/ul/li[' str(box) ']/span[1]'
WebDriverWait(driver, 10).until(EC.presence_of_element_located(el)
el.click()
time.sleep(1.5)
#for i in tqdm(range(1,2500)):
while not check_for_break_condition(driver):
try:
WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.XPATH, '//*[@id="data_loader"]')))
driver.execute_script("stnyEvent.getNextPage();")
except:
captureError(driver)
el.click()
uj5u.com熱心網友回復：
試試這個。看起來您可以通過發布請求獲得它。
import requests
import pandas as pd
from bs4 import BeautifulSoup
import re
import time
s = requests.Session()
s.get('https://www.seethroughny.net/')
cookies = s.cookies.get_dict()
cookieStr = ''
for k,v in cookies.items():
cookieStr = f'{k}={v};'
url = 'https://www.seethroughny.net/tools/required/reports/payroll'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
'cookie':cookieStr,
'accept': 'application/json, text/javascript, */*; q=0.01',
'referer': 'https://www.seethroughny.net/payrolls'}
payload = {'action':'get'}
def get_data(s, formData, payload):
retry = 0
try:
jsonData = s.post(url, headers=headers, data=formData, params=payload, timeout=10).json()
success = True
except:
retry =1
print('Retry: %s' %retry)
jsonData = None
success = False
time.sleep(10)
return jsonData, success
def parse_data(jsonData):
rows = []
html = jsonData['html']
soup = BeautifulSoup(html, 'html.parser')
trs = soup.find_all('tr', {'id':re.compile("^resultRow")})
for tr in trs:
tds = tr.find_all('td')
row = {
'Name':tds[1].text,
'Employer/Agency':tds[2].text,
'Total Pay':tds[3].text,
'Subagency/Type':tds[4].text
}
expandedRow = tr.find_next('tr', {'id':re.compile("^expandRow")})
td = expandedRow.find_all('td')[1]
divs = td.find_all('div', {'class':re.compile("^row")})
for each in divs:
header = each.find_all('div')[0].text
value = each.find_all('div')[1].text
row.update({header:value})
rows.append(row)
return rows
rows = []
for year in [2021]:
completedYear = False
# Get Total pages
formData = {
'PayYear[]': f'{year}',
'SortBy': 'YTDPay DESC',
'current_page': '0',
'result_id': '0',
'url': '/tools/required/reports/payroll?action=get',
'nav_request': '0'}
jsonData, success = get_data(s, formData, payload)
total_pages = jsonData['total_pages']
print(f'Year: {year}\tPage: 0 of {total_pages}')
rows = parse_data(jsonData)
page = 1
while completedYear == False:
success = False
while success == False:
formData.update({'current_page': '%s' %page})
jsonData, success = get_data(s, formData, payload)
rows = parse_data(jsonData)
print(f'Year: {year}\tPage: {page} of {total_pages}')
page =1
if page > total_pages:
completedYear = True
df = pd.DataFrame(rows)
輸出：
print(df.head(5).to_string())
Name Employer/Agency Total Pay Subagency/Type SubAgency/Type Title Rate of Pay Pay Year Pay Basis Branch/Major Category
0 Johnson, Candace Roswell Park Cancer Institute Corporation $1,622,807 Roswell Park Cancer Institute Corporation Roswell Park Cancer Institute Corporation President & Ceo $0 2021 Annual Public Authorities
1 Kuettel, Michael Roswell Park Cancer Institute Corporation $958,373 Roswell Park Cancer Institute Corporation Roswell Park Cancer Institute Corporation Chair $0 2021 Annual Public Authorities
2 Odunsi, Adekunle Roswell Park Cancer Institute Corporation $918,079 Roswell Park Cancer Institute Corporation Roswell Park Cancer Institute Corporation Deputy Director $0 2021 Annual Public Authorities
3 Fenstermaker, Robert Roswell Park Cancer Institute Corporation $914,887 Roswell Park Cancer Institute Corporation Roswell Park Cancer Institute Corporation Chair $0 2021 Annual Public Authorities
4 Guru, Khurshid Roswell Park Cancer Institute Corporation $856,979 Roswell Park Cancer Institute Corporation Roswell Park Cancer Institute Corporation Chair $0 2021 Annual Public Authorities
轉載請註明出處，本文鏈接：https://www.uj5u.com/caozuo/421035.html
標籤：