我正在開發一個網路爬蟲來從 Solis Pro 平臺（Ginlong）挖掘資料，但我在從用戶注冊的植物中獲取異步資料時遇到問題。我正在使用 Selenium bs4 并且發生了以下情況。網址是https://m.ginlong.com/pro/epc/plantview/view/doAsyncPlantList.json。我發送了一個有效負載，理論上我應該接收資料，但我要么收到錯誤，要么只收到部分資料（僅 {status: 1}）。
平臺和有效載荷
url = 'https://m.ginlong.com/'
chrome_options = Options()
chrome_options.add_argument("--headless")
webdriver = Chrome(ChromeDriverManager().install(), options=chrome_options)
webdriver.get(url 'login.html')
webdriver.find_element(By.XPATH, '//button[@ng-click="typeChange()"]').click()
webdriver.find_element_by_name("username").send_keys(username)
webdriver.find_element_by_name("password").send_keys(password)
#webdriver.find_element(By.XPATH, '//button[@]').click()
webdriver.find_element(By.XPATH, '//button[@id="login"]').click()
soup = BeautifulSoup(webdriver.page_source, 'lxml')
headers = {
"content-type": "application/x-www-form-urlencoded",
"accept": "application/json",
"user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Mobile Safari/537.36"
}
postData = {
"plantArray": [{"plantId":1033610,"timezoneId":24,"updateTime":"2022-01-18"},{"plantId":1028383,"timezoneId":24,"updateTime":"2022-01-18"}]
}
response = webdriver.request('POST', url 'pro/epc/plantview/view/doAsyncPlantList.json', headers=headers, data=postData)
if response.status_code != 200:
raise RuntimeError("A requisi??o falhou: %s", response)
data = BeautifulSoup(response.text, 'html.parser')
print(data.text)
我得到的錯誤
Exception: Unrecognized token 'plantId': was expecting ('true', 'false' or 'null')
at [Source: plantId,timezoneId,updateTime,plantId,timezoneId,updateTime; line: 1, column: 8]
com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'plantId': was expecting ('true', 'false' or 'null')
at [Source: plantId,timezoneId,updateTime,plantId,timezoneId,updateTime; line: 1, column: 8]
at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1702)
at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:558)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2836)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:1899)
at com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:749)
at com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:3847)
at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:3792)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2835)
at com.igen.solarman.controller.epc.PlantViewController.doAsyncPlantList(PlantViewController.java:345)
at com.igen.solarman.controller.epc.PlantViewController$$FastClassBySpringCGLIB$$70060edf.invoke()
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:720)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:655)
at com.igen.solarman.controller.epc.PlantViewController$$EnhancerBySpringCGLIB$$b39c8e9a.doAsyncPlantList()
at sun.reflect.GeneratedMethodAccessor1121.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:221)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:136)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:114)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:963)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:872)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at com.alibaba.druid.support.http.WebStatFilter.doFilter(WebStatFilter.java:123)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.tuckey.web.filters.urlrewrite.RuleChain.handleRewrite(RuleChain.java:176)
at org.tuckey.web.filters.urlrewrite.RuleChain.doRules(RuleChain.java:145)
at org.tuckey.web.filters.urlrewrite.UrlRewriter.processRequest(UrlRewriter.java:92)
at org.tuckey.web.filters.urlrewrite.UrlRewriteFilter.doFilter(UrlRewriteFilter.java:394)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:61)
at org.apache.shiro.web.servlet.AdviceFilter.executeChain(AdviceFilter.java:108)
at org.apache.shiro.web.servlet.AdviceFilter.doFilterInternal(AdviceFilter.java:137)
at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
at org.apache.shiro.web.servlet.ProxiedFilterChain.doFilter(ProxiedFilterChain.java:66)
at org.apache.shiro.web.servlet.AbstractShiroFilter.executeChain(AbstractShiroFilter.java:449)
at org.apache.shiro.web.servlet.AbstractShiroFilter$1.call(AbstractShiroFilter.java:365)
at org.apache.shiro.subject.support.SubjectCallable.doCall(SubjectCallable.java:90)
at org.apache.shiro.subject.support.SubjectCallable.call(SubjectCallable.java:83)
at org.apache.shiro.subject.support.DelegatingSubject.execute(DelegatingSubject.java:383)
at org.apache.shiro.web.servlet.AbstractShiroFilter.doFilterInternal(AbstractShiroFilter.java:362)
at org.apache.shiro.web.servlet.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:125)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:262)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.logging.log4j.web.Log4jServletFilter.doFilter(Log4jServletFilter.java:71)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:494)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1137)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
資料
有沒有人嘗試獲取此資料或知道如何解決此錯誤？
uj5u.com熱心網友回復：
改變： response = webdriver.request('POST', url 'pro/epc/plantview/view/doAsyncPlantList.json', headers=headers, data=postData)
對此： response = webdriver.request('POST', url 'pro/epc/plantview/view/doAsyncPlantList.json', headers=headers, json=json.dumps(postData))
（記得匯入json）:)
我不確定為什么會這樣，但確實如此，為了進一步閱讀，請參閱關于和之間區別的討論data=：json=Python請求包中的資料和json引數之間的差異
另外，我已經設法讓它只處理應該加快速度的請求，請注意，我必須在最后將我的網址更改為“cpro”而不是“pro”，因為我沒有專業人士帳戶：“https://m.ginlong.com/pro/epc/plantview/view/doAsyncPlantList.json”
import requests
import json
s = requests.Session()
headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'}
url = 'https://m.ginlong.com/login.html'
resp = s.get(url,headers=headers)
print(resp)
login_url = 'https://m.ginlong.com/cpro/login/validateLogin.json'
payload = {
'userName':'username',
'userNameDisplay':'username',
'password':'password',
'lan':'2',
'domain':'m.ginlong.com',
'userType':'C'
}
headers = {
'accept':'application/json, text/plain, */*',
'accept-encoding':'gzip, deflate, br',
'content-type':'application/x-www-form-urlencoded',
'origin':'https://m.ginlong.com',
'referer':'https://m.ginlong.com/login.html',
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
login = s.post(login_url,headers=headers,data=payload)
print(login)
new_plant_payload = {
'name':'new_plant_2',
'nmi':'',
'type':'2',
'gridType':'1',
'searchInput':'',
'positionAddressInput':'ChinaJiang Su ShengWu Xi ShiXin Wu QuTian An Zhi Hui Cheng A2',
'rectangle':'120.36118,31.5065;',
'lon':'120.36118',
'lat':'31.5065',
'address':'ChinaJiang Su ShengWu Xi ShiXin Wu QuTian An Zhi Hui Cheng A2',
'country':'Jiang Su Sheng',
'state':'Wu Xi Shi',
'city':'Xin Wu Qu',
'district':'Tian An Zhi Hui Cheng A2',
'countryId':'44',
'stateId':'',
'cityId':'',
'districtId':'',
'power':'5.00',
'timezoneId':'34',
'direction':'180°',
'angle':'26°',
'currencyId':'19',
'benchmarkPrice':'1.372',
'price':'0.75',
'priceNet':'1',
'subsidy':'0.372',
'subsidyYears':'20',
'subsidyLocal':'0',
'subsidyLocalYears':'0',
'subsidyCity':'0',
'subsidyCityYears':'0',
'subsidyCounty':'0',
'subsidyCountyYears':'0',
'cost':'37500.00',
'subsidyBuild':'1000',
'percent':'0%',
'interest':'0',
'years':'',
'repay':'1',
'phone':''
}
create_plant_headers = {
'accept':'application/json',
'content-type':'application/x-www-form-urlencoded',
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
create_url = 'https://m.ginlong.com/cpro/epc/plantCreate/create/doSubmitPlant.json'
create = s.post(create_url,headers=create_plant_headers,data=new_plant_payload)
print(create)
print(create.text)
plant_id = create.json()['result']
async_url = 'https://m.ginlong.com/cpro/epc/plantview/view/doAsyncPlantList.json'
postData = {
"plantArray": [{"plantId":plant_id,"timezoneId":24,"updateTime":"2022-01-18"}]
}
post_async = s.post(async_url,json=json.dumps(postData))
print(post_async)
print(post_async.text)
轉載請註明出處，本文鏈接：https://www.uj5u.com/qukuanlian/417670.html
標籤：
上一篇：Python和beautifoulsoup的編碼問題
下一篇：.HTML文本的遍歷決議