所以我的想法是，如果我添加一些可以將 url-range 拆分為 5 的東西，然后為 5 個 chromedriver 實體中的每一個實體分配它們自己的 url-range 拆分來處理它，那么抓取速度會更快。那是我最大的問題。但是，如果每個 chromedriver 都有自己的 csv 檔案，那可能會更好，或者我需要添加一些東西來將所有的抓取合并到一個檔案中？我真的很茫然，我已經在提升我的技能水平了。我永遠感激任何關于如何讓多執行緒作業的具體幫助。謝謝！

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
import csv
path_to_file = "test1.csv"
csvFile = open(path_to_file, 'a', encoding="utf-8", newline='')
csvWriter = csv.writer(csvFile)
options = webdriver.ChromeOptions() 
driver = webdriver.Chrome(options=options)
header_added = False
time.sleep(3)
for i in range(1,153512):
    print(f"https://www.ancestry.com/discoveryui-content/view/{i}:61965")
    driver.get(f"https://www.ancestry.com/discoveryui-content/view/{i}:61965")
    try:
        Name = driver.find_element(By.XPATH,"//table[@id='recordServiceData']//tr[contains(.,'Name:')]").text.replace("Name:", "")
    except:
        Name =''
    csvWriter.writerow([i, Name])
    print(Name)

uj5u.com熱心網友回復：

嘗試這個：

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
import csv
path_to_file = "test1.csv"
csvFile = open(path_to_file, 'a', encoding="utf-8", newline='')
csvWriter = csv.writer(csvFile)

header_added = False
time.sleep(3)


    
def init_driver_worker(_range_task): #create new instace of chrome then make it do its job
    ##### init driver
    options = webdriver.ChromeOptions()
    #you can't run multible instances of chrome
    #  with the same profile being used,
    #  so either create new profile for each instance or use incognito mode
    options.add_argument("--incognito")
    options.add_argument("--headless") #use headless browser (no GUI) to be faster
    driver = webdriver.Chrome(options=options)
    ##### do the task
    for i in _range_task:
        print(f"https://www.ancestry.com/discoveryui-content/view/{i}:61965")
        driver.get(f"https://www.ancestry.com/discoveryui-content/view/{i}:61965")
        try:
            Name = driver.find_element(By.XPATH,"//table[@id='recordServiceData']//tr[contains(.,'Name:')]").text.replace("Name:", "")
        except:
            Name =''
        csvWriter.writerow([i, Name])
        print(Name)
    exit() #close the thread
    
    
    
def split_range(_range, parts): #split a range to chunks
    chunk_size = int(len(_range)/parts)
    chunks = [_range[x:x chunk_size] for x in range(0, len(_range), chunk_size)]
    return chunks

my_range = range(1,153512)
chunks = split_range(my_range, 10) # split the task to 10 instances of chrome

from threading import Thread
thread_workers = []
for chunk in chunks:
    t = Thread(target=init_driver_worker, args=([chunk]))
    thread_workers.append(t)
    t.start()
    
# wait for the thread_workers to finish
for t in thread_workers:
    t.join()

標籤：Python 硒 python-多线程

上一篇：在Seleniumpython中測驗WebScrape而不運行整個腳本？

下一篇：硒|無法定位輸入元素