我正在嘗試從公司資料庫中抓取公司資訊。我有一個文本檔案中的公司串列，我希望 selenium 進入網站的搜索并一一抓取所需的資訊。
我的問題是，由于某種原因，它只輸入串列中的姓氏。我怎么能告訴python刮掉串列上的第一個公司名稱，然后是下一個，依此類推？
我的代碼如下：
# -*- coding: utf-8 -*-
# from typing_extensions import Self
from lib2to3.pgen2 import driver
import scrapy
from scrapy.selector import Selector
# from scrapy_selenium import SeleniumRequest
from time import sleep
from turtle import delay
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from shutil import which
count = 0
file = open ("cegek.txt", "r")
lines = file.readlines()
for line in lines:
count = 1
# # cegek = "1000 út Kft."
class HtSpiderSeleniumceg(scrapy.Spider):
name = 'kamara'
allowed_domains = ["wwww.ceginfo.hu/"]
start_urls = [
'https://www.ceginfo.hu'
]
def __init__(self):
chrome_options = Options()
# chrome_options.add_argument("--headless")
#get login page
driver = webdriver.Chrome(executable_path="./chromedriver", options=chrome_options)
driver.get("https://www.ceginfo.hu/")
driver.find_element_by_xpath("//input[@type='search']").send_keys(line)
sleep(2)
driver.find_element_by_xpath("//input[@type='search']").send_keys(u'\ue007')
self.html = driver.page_source
driver.close()
#scrape needed info
def parse(self, response):
resp = Selector(text=self.html)
for ceg in resp.xpath("(//div[contains(@class, 'd-flex flex-column flex-sm-row justify-content-between align-items-center')])[1]"):
yield {
'cegnev': ceg.xpath("(//h2[contains(@class,'s-title heading')])[1]/text()").get(),
'adoszam': ceg.xpath("(.//span[@class='text-uppercase c-border me-lg-3'])[1]/text()").get(),
'cegjegy': ceg.xpath("(.//span[@class='c-border'])[1]/text()").get()
}
這是公司名稱串列的確切格式：
SZIMIKRON Ipari Kft.
Tigra Computer- és Irodatechnikai Kft.
Tradeland Kft.
T?r?k László EV T?r?k Kulcsszervíz
Tungsram Operations Kft.
Tutti élelmiszeripari Kft.
Water and Soil Kft.
Webkey Development Kft.
ZDMnet
在一些幫助下，現在搜索串列中的第一個名字，但由于錯誤，蜘蛛沒有抓取和中斷：
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=100.0.4896.60)
這是我的新代碼，底部的 #out 部分顯示了我的新問題的假定解決方案，但我不知道如何實作它，我嘗試將它放在不同的地方但它不起作用。另外，我不確定“your_element”部分指的是什么，這個解決方案是在這個執行緒中提出的： StaleElementReferenceException on Python Selenium
# -*- coding: utf-8 -*-
# from typing_extensions import Self
from lib2to3.pgen2 import driver
import scrapy
from scrapy.selector import Selector
# from scrapy_selenium import SeleniumRequest
from time import sleep
from turtle import delay
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.chrome.options import Options
from shutil import which
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import StaleElementReferenceException
# # cegek = "1000 út Kft."
class HtSpiderSeleniumceg(scrapy.Spider):
name = 'kamara'
allowed_domains = ["wwww.ceginfo.hu/"]
start_urls = [
'https://www.ceginfo.hu'
]
global names_to_search
names_to_search = open("cegek.txt", "r").readlines()
def __init__(self):
chrome_options = Options()
# chrome_options.add_argument("--headless")
self.driver = webdriver.Chrome(executable_path="./chromedriver", options=chrome_options)
self.driver.get("https://ceginfo.hu/ceglista/cegek")
sleep(2)
self.start_urls = [self.driver.current_url]
sleep(2)
global this_driver
this_driver = self.driver.find_element_by_xpath("//input[@type='search']")
this_driver.send_keys(names_to_search[0])
sleep(2)
this_driver.send_keys(u'\ue007')
sleep(5)
def parse(self, response):
self.driver.get(response.url)
print('this_driver')
print(this_driver)
print('names_to_search')
print(names_to_search)
for names in names_to_search:
print('searching this names:')
print(names)
resp = Selector(text=self.driver.page_source)
sleep(5)
for ceg in resp.xpath("(//p[@class='mb-3 m-sm-0 meta d-flex flex-column flex-lg-row me-auto'])[1]"):
yield {
'cegnev': ceg.xpath("(//h2[contains(@class,'s-title heading')])[1]/text()").get(),
'adoszam': ceg.xpath("(.//span[@class='text-uppercase c-border me-lg-3'])[1]/text()").get(),
'cegjegy': ceg.xpath("(.//span[@class='c-border'])[1]/text()").get()
}
try:
print(this_driver)
this_driver.send_keys(names)
# driver.find_element_by_xpath("//input[@type='search']").send_keys(line)
sleep(2)
this_driver.send_keys(u'\ue007')
except:
print('exception - do not break')
self.driver.close()
# my_element_id = "(//p[@class='mb-3 m-sm-0 meta d-flex flex-column flex-lg-row me-auto'])[1]"
# ignored_exceptions=(NoSuchElementException,StaleElementReferenceException,)
# your_element = WebDriverWait(self.driver, 20,ignored_exceptions=ignored_exceptions)\
# .until(expected_conditions.presence_of_element_located((By.XPATH, my_element_id)))
uj5u.com熱心網友回復：
如果不安裝 Selenium、Web 驅動程式等，我無法完全復制您的代碼，但這就是您實施解決方案的方式。
撰寫一個函式從 cegek.txt 讀取名稱并附加到串列中：
names_to_search = []
def get_names_to_search():
# open file to read
file = open ("cegek.txt", "r")
# read lines in file
lines = file.readlines()
# loop through file and append names to list
for line in lines:
names_to_search.append(line.strip())
# The names_to_search list will contain:
['SZIMIKRON Ipari Kft.', 'Tigra Computer- és Irodatechnikai Kft.', 'Tradeland Kft.', 'T?r?k László EV T?r?k Kulcsszervíz', 'Tungsram Operations Kft.', 'Tutti élelmiszeripari Kft.', 'Water and Soil Kft.', 'Webkey Development Kft.', 'ZDMnet']
回圈names_to_search并將每個名稱傳遞給driver.find_element_by_xpath("//input[@type='search']").send_keys(name)
for name in names_to_search:
driver.find_element_by_xpath("//input[@type='search']").send_keys(name)
轉載請註明出處，本文鏈接：https://www.uj5u.com/net/455135.html