我正在嘗試使用 4 個下拉選單從以下網站抓取資料 - 單擊每個下拉選單后，它們會顯示一個表格，我想從中抓取資料。我想合并所有下拉選單中所有表格的資訊。
我正在使用RSelenium包，但是因為我對網路抓取非常陌生，我無法理解如何使用四個可用選項進行回圈以獲得最終表。
https://hindi.iocl.com/lpgdistributors.aspx
我嘗試了之前關于網頁抓取的討論并相應地修改了代碼。
library(RSelenium)
library(rvest)
system("taskkill /im java.exe /f", intern=FALSE, ignore.stdout=FALSE)
rD <- rsDriver(browser = c("firefox")) #specify browser type you want Selenium to open
remDr <- rD$client
remDr$navigate("https://hindi.iocl.com/lpgdistributors.aspx") # navigates to webpage
# select first dropdown list
option <- remDr$findElement(using='id', value="cmbState")
#get all option values from dropdown list
option_values <- option$getPageSource()[[1]] %>%
str_extract_all("1[0-9]{3}")
# select 2nd dropdown list
option <- remDr$findElement(using='id', value="cmbDistrict")
#get all option values from dropdown list
option_values <- option$getPageSource()[[1]] %>%
str_extract_all("1[0-9]{3}")
# select 3rd dropdown list
option <- remDr$findElement(using='id', value="cmbMarket")
#get all option values from dropdown list
option_values <- option$getPageSource()[[1]] %>%
str_extract_all("1[0-9]{3}")
#select 4th dropdown list
option2 <- remDr$findElement(using='id', value="cmbArea")
#get all option values from dropdown list
option_values_2 <- option2$getElementText() %>%
str_split("\\n") %>%
unlist()
#### create loop to loop over all tables...
option <- remDr$findElement(using='id', value="cmbState")
option <- remDr$findElement(using = 'xpath', "//*/option[@value = '1']") #change '1194' to values in option_values in loop
option$clickElement()
# change dropdown selection
option2 <- remDr$findElement(using='id', value="cmbDistrict")
option2 <- remDr$findElement(using = 'xpath', "//*/option[@value = '185']") #change 'AHB' to values in option_values_2 in loop
option2$clickElement()
# change dropdown selection
option3 <- remDr$findElement(using='id', value="cmbMarket")
option3 <- remDr$findElement(using = 'xpath', "//*/option[@value = '2314']") #change 'AHB' to values in option_values_2 in loop
option3$clickElement()
# change dropdown selection
option4 <- remDr$findElement(using='id', value="cmbArea")
option4 <- remDr$findElement(using = 'xpath', "//*/option[@value = '57']") #change 'AHB' to values in option_values_2 in loop
option4$clickElement()
# click submit
submit <- remDr$findElement(using='id', value="btnSearch")
submit$clickElement()
#get table
tb <- remDr$findElement(using='id', value="grdDistributors")
tb$getPageSource()[[1]] %>%
read_html() %>%
html_table(fill = TRUE)
uj5u.com熱心網友回復：
這是使用的部分解決方案RSelenium，
library(RSelenium)
driver <- rsDriver(browser = "chrome")
remDr<-driver[["client"]]
remDr$navigate("https://hindi.iocl.com/lpgdistributors.aspx")
獲取所有狀態的串列
webElem <- remDr$findElement(using='xpath', value= '//*[@id="cmbState"]')
states <- webElem$getElementText()
states= unlist(states)
#removing line breakers and converting to vector
states = sapply(strsplit(states, split='\n ', fixed=TRUE), `[`)
[,1]
[1,] " Andaman & Nicobar"
[2,] "Andhra Pradesh"
[3,] "Arunachal Pradesh"
[4,] "Assam"
[5,] "Bihar"
[6,] "Chandigarh"
[7,] "Chhatisgarh"
[8,] "Goa"
[9,] "Gujarat"
[10,] "Haryana"
[11,] "Himachal Pradesh"
[12,] "Jammu & Kashmir"
[13,] "Jharkhand"
[14,] "Karnataka"
現在您必須遍歷各州以獲取所有地區的串列，例如，
### Select the State
remDr$navigate("https://hindi.iocl.com/lpgdistributors.aspx")
opt1 <- remDr$findElement(using='xpath', value= '//*[@id="cmbState"]')
opt1$clickElement()
opt1$sendKeysToElement(list('Karnataka'))
### Get list of all the districts
webElem <- remDr$findElement(using='xpath', value= '//*[@id="cmbDistrict"]')
webElem$clickElement()
district <- webElem$getElementText()
district = unlist(district)
district = sapply(strsplit(district, split='\n ', fixed=TRUE), `[`)
[,1]
[1,] " Bagalkot"
[2,] "Bangalore"
[3,] "Belgaum"
[4,] "Bellary"
[5,] "Bidar"
[6,] "Bijapur"
[7,] "Chamrajnagar"
[8,] "Chickmagalur"
[9,] "Chitradurga"
[10,] "Coorg"
[11,] "Dakshina Kannada"
[12,] "Davangere"
[13,] "Dharwad"
### Get list of all the markets
webElem <- remDr$findElement(using='xpath', value= '//*[@id="cmbMarket"]')
webElem$clickElement()
market <- webElem$getElementText()
market = unlist(market)
market = sapply(strsplit(market, split='\n ', fixed=TRUE), `[`)
[,1]
[1,] " Afzalpur"
[2,] "Ajjampur"
[3,] "Aland"
[4,] "Ankola"
[5,] "B. Mathikere"
[6,] "Bagalkot"
[7,] "Bailhongal"
[8,] "Bandipur"
[9,] "Bangalore"
[10,] "Basavakalyan"
[11,] "Belgaum"
[12,] "Bellary"
[13,] "Belthangadi"
[14,] "Bhadravathi"
[15,] "Bhadravati"
[16,] "Bhatkal"
[17,] "Bidar"
[18,] "Bijapur"
[19,] "Challakere"
uj5u.com熱心網友回復：
您可以輕松使用RSelenium，但在您必須檢查下拉選單的所有值之前。
在創建示例之前，有必要了解要與下拉選單互動，您必須研究頁面背后的 html 代碼并了解選單的每個元素。
這是一個例子：
library(RSelenium)
#Selenium environment activation
rD <- rsDriver(browser = "firefox", check = FALSE)
remDr <- rD[["client"]]
remDr$navigate("https://hindi.iocl.com/lpgdistributors.aspx")
# We collect all elements that are present in the drop menu and we insert in a list
menu_1<-list("Assam")
menu_2<-list("Goalpara")
menu_3<-list("Goalpara")
menu_4<-list("Guwahati")
# We create a four for loops with two focus:
# 1. by click we active the drop menu
# 2. in this case we sent the text (the format should be factor) to choose
for( i in menu_1) {
first_element<-as.factor(i)
remDr$findElement(using = 'xpath', value = '//*[@id="cmbState"]')$clickElement()
remDr$findElement(using = 'xpath', value = '//*[@id="cmbState"]')$sendKeysToElement(list(first_element))
Sys.sleep(1) #Is better to implement a sleep
for( j in menu_2) {
first_element<-as.factor(j)
remDr$findElement(using = 'xpath', value = '//*[@id="cmbDistrict"]')$clickElement()
remDr$findElement(using = 'xpath', value = '//*[@id="cmbDistrict"]')$sendKeysToElement(list(first_element))
Sys.sleep(1) #Is better to implement a sleep
for( e in menu_3) {
first_element<-as.factor(e)
remDr$findElement(using = 'xpath', value = '//*[@id="cmbMarket"]')$clickElement()
remDr$findElement(using = 'xpath', value = '//*[@id="cmbMarket"]')$sendKeysToElement(list(first_element))
Sys.sleep(1) #Is better to implement a sleep
for( f in menu_4) {
first_element<-as.factor(f)
remDr$findElement(using = 'xpath', value = '//*[@id="cmbArea"]')$clickElement()
remDr$findElement(using = 'xpath', value = '//*[@id="cmbArea"]')$sendKeysToElement(list(first_element))
Sys.sleep(1) #Is better to implement a sleep
}
}
}
}
# Click the button for searching
remDr$findElement(using = 'xpath', value = '//*[@id="btnSearch"]')$clickElement()
轉載請註明出處，本文鏈接：https://www.uj5u.com/net/317844.html