如何用同一類刮掉另一個跨度-有解無憂

我使用 beautifulSoup4，這是我的代碼：

def extract(page):
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/{page}/'
r = requests.get(url)
soup = BeautifulSoup(r.content,'html.parser')
return soup

def transform(soup):
divs = soup.find_all('div', class_ = 'sx2jih0 zcydq876 zcydq866 zcydq896 zcydq886 zcydq8n zcydq856 zcydq8f6 zcydq8eu')
for items in divs:
location = items.find('span', attrs={'class': 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'}).text.strip()

salary = items.find_next_sibling('span', attrs={'class': 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'}).text.strip()

兩個跨度都具有相同的類，但是當我報廢時，兩個結果都是相同的。

uj5u.com熱心網友回復：

獲取所有跨度find_all，然后洗掉它們少于 2 個跨度的專案。然后從 spans 串列中獲取location和salary

import requests
from bs4 import BeautifulSoup

def extract(page):
    url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/{page}/'
    r = requests.get(url)
    soup = BeautifulSoup(r.content,'html.parser')
    # transform
    jobs = soup.find_all('div', class_ = 'sx2jih0 zcydq876 zcydq866 zcydq896 zcydq886 zcydq8n zcydq856 zcydq8f6 zcydq8eu')
    data = []
    for job in jobs[:100]:
        items = job.find_all('span', attrs = {'class' : 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'})
        if len(items) < 2:
            continue
        location = items[0].text.strip()
        salary = items[1].text.strip()

        data.append([location, salary])
    
    return data
        
data = extract(1)
print(data)

uj5u.com熱心網友回復：

盡量避免通過 HTML 中的動態部分（例如類）進行選擇 - 雖然您仍然想出可以使用的兄弟關系find_next_sibling('span')。

以下將僅按標簽選擇，并檢查是否有薪水以避免錯誤但刮掉整個作業，同時顯示薪水None：

for item in soup.select('article'):
    data.append({
        'title': item.h1.text,
        'location': item.h1.parent.next_sibling.text,
        'salary': item.h1.parent.next_sibling.find_next_sibling('span').text if item.h1.parent.next_sibling.find_next_sibling('span') else None,
    })

例子

import requests
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/1/'
r = requests.get(url)
soup = BeautifulSoup(r.content, 'html.parser')
data=[]
for item in soup.select('article'):
    data.append({
        'title': item.h1.text,
        'location': item.h1.parent.next_sibling.text,
        'salary': item.h1.parent.next_sibling.find_next_sibling('span').text if item.h1.parent.next_sibling.find_next_sibling('span') else None,
    })
data

uj5u.com熱心網友回復：

使用動態類不是最好的主意。該網站使用 api 和 graphql 進行求職。

import requests
import json
import pandas as pd


def foo(page, keyword):
    url = "https://xapi.supercharge-srp.co/job-search/graphql?country=my&isSmartSearch=true"
    payload = json.dumps({
      "query": "query getJobs($country: String, $locale: String, $keyword: String, $createdAt: String, $jobFunctions: [Int], $categories: [String], $locations: [Int], $careerLevels: [Int], $minSalary: Int, $maxSalary: Int, $salaryType: Int, $candidateSalary: Int, $candidateSalaryCurrency: String, $datePosted: Int, $jobTypes: [Int], $workTypes: [String], $industries: [Int], $page: Int, $pageSize: Int, $companyId: String, $advertiserId: String, $userAgent: String, $accNums: Int, $subAccount: Int, $minEdu: Int, $maxEdu: Int, $edus: [Int], $minExp: Int, $maxExp: Int, $seo: String, $searchFields: String, $candidateId: ID, $isDesktop: Boolean, $isCompanySearch: Boolean, $sort: String, $sVi: String, $duplicates: String, $flight: String, $solVisitorId: String) {\n  jobs(\n    country: $country\n    locale: $locale\n    keyword: $keyword\n    createdAt: $createdAt\n    jobFunctions: $jobFunctions\n    categories: $categories\n    locations: $locations\n    careerLevels: $careerLevels\n    minSalary: $minSalary\n    maxSalary: $maxSalary\n    salaryType: $salaryType\n    candidateSalary: $candidateSalary\n    candidateSalaryCurrency: $candidateSalaryCurrency\n    datePosted: $datePosted\n    jobTypes: $jobTypes\n    workTypes: $workTypes\n    industries: $industries\n    page: $page\n    pageSize: $pageSize\n    companyId: $companyId\n    advertiserId: $advertiserId\n    userAgent: $userAgent\n    accNums: $accNums\n    subAccount: $subAccount\n    minEdu: $minEdu\n    edus: $edus\n    maxEdu: $maxEdu\n    minExp: $minExp\n    maxExp: $maxExp\n    seo: $seo\n    searchFields: $searchFields\n    candidateId: $candidateId\n    isDesktop: $isDesktop\n    isCompanySearch: $isCompanySearch\n    sort: $sort\n    sVi: $sVi\n    duplicates: $duplicates\n    flight: $flight\n    solVisitorId: $solVisitorId\n  ) {\n    total\n    totalJobs\n    relatedSearchKeywords {\n      keywords\n      type\n      totalJobs\n    }\n    solMetadata\n    suggestedEmployer {\n      name\n      totalJobs\n    }\n    queryParameters {\n      key\n      searchFields\n      pageSize\n    }\n    experiments {\n      flight\n    }\n    jobs {\n      id\n      adType\n      sourceCountryCode\n      isStandout\n      companyMeta {\n        id\n        advertiserId\n        isPrivate\n        name\n        logoUrl\n        slug\n      }\n      jobTitle\n      jobUrl\n      jobTitleSlug\n      description\n      employmentTypes {\n        code\n        name\n      }\n      sellingPoints\n      locations {\n        code\n        name\n        slug\n        children {\n          code\n          name\n          slug\n        }\n      }\n      categories {\n        code\n        name\n        children {\n          code\n          name\n        }\n      }\n      postingDuration\n      postedAt\n      salaryRange {\n        currency\n        max\n        min\n        period\n        term\n      }\n      salaryVisible\n      bannerUrl\n      isClassified\n      solMetadata\n    }\n  }\n}\n",
      "variables": {
        "keyword": keyword,
        "jobFunctions": [],
        "locations": [],
        "salaryType": 1,
        "jobTypes": [],
        "createdAt": None,
        "careerLevels": [],
        "page": page,
        "country": "my",
        "categories": [],
        "workTypes": [],
        "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
        "industries": [],
        "locale": "en"
      }
    })
    headers = {
        'accept': '*/*',
        'content-type': 'application/json'
    }

    response = requests.request("POST", url, headers=headers, data=payload)
    salary = "Not specified"
    result = []
    for job in (response.json()['data']['jobs']['jobs']):
        locations = []
        if job['salaryVisible']:
            salary = f"{job['salaryRange']['min']}-{job['salaryRange']['max']} {job['salaryRange']['currency']}"
        for location in job['locations']:
            locations.append(location['name'])
        result.append([job['jobTitle'], job['description'], job['jobUrl'], salary, ", ".join(locations)])
    df = pd.DataFrame(result)
    df.columns = ['Title', 'Description', 'Link', 'Salary', 'Location']
    print(df.to_string())


foo(1, "personal assistant")

輸出：

                                                                         Title                                                                                                                                                Description                                                                                                                                                                                                                                                   Link           Salary                Location
0                                                Personal Assistant/ Secretary        Job Description:GETO GLOBAL CONSTRUCTION TECH MALAYSIA SDN. BHD. is a subsidiary of a public listed company from China, specializing in Aluminum...                                                                                     https://www.jobstreet.com.my/en/job/personal-assistant-secretary-4986870?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=1&jobId=jobstreet-my-job-4986870    3000-6000 MYR        Shah Alam/Subang
1                                                          Executive Assistant      Responsibilities:  Responsible in managing day-to-day office related and administrative tasks. Organizing and attending meetings and ensuring well...                                                                                              https://www.jobstreet.com.my/en/job/executive-assistant-4986449?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=2&jobId=jobstreet-my-job-4986449    2000-3000 MYR          Johor - Others
2                                                                    Secretary        A good write-up Secretary's resume is a Must with experience and can handle Managing Director company matters only. Fresh grads or many years in...                                                                                                        https://www.jobstreet.com.my/en/job/secretary-4986714?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=3&jobId=jobstreet-my-job-4986714    2000-3000 MYR           Petaling Jaya
3                                                 Secretary/Personal assistant  We are looking for Secretary/Personal assistant in our company for: Organising and maintaining office systems Screening phone calls, emails, enquiries...                                                                                     https://www.jobstreet.com.my/en/job/secretary-personal-assistant-4983131?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=4&jobId=jobstreet-my-job-4983131    2000-3000 MYR            Kuala Lumpur
4                                         Personal Assistant to Country Leader    Do you want to make a positive impact on the world? Sustainable development is at the heart of everything we do.Seventy-five years ago, we created a...                                                                             https://www.jobstreet.com.my/en/job/personal-assistant-to-country-leader-4986906?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=5&jobId=jobstreet-my-job-4986906    2000-3000 MYR           Petaling Jaya
5                                        Personal Assistant to Senior Director  Requirement: 5 – 8 years in handling PA end to end tasks for a Director/CEO/Managing Director Highly proficient in MS Office software Strong knowledge...                                                                            https://www.jobstreet.com.my/en/job/personal-assistant-to-senior-director-4986792?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=6&jobId=jobstreet-my-job-4986792    2000-3000 MYR        Shah Alam/Subang
6                                                           Personal Assistant     Report to top management; Attend to all aspects of PA and corporate work; Draft and manage correspondences, email, and memos; Schedule and organize...                                                                                               https://www.jobstreet.com.my/en/job/personal-assistant-4987086?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=7&jobId=jobstreet-my-job-4987086    2000-4000 MYR                 Puchong
7                                   Personal Assistant (Secretary) to Director       Responsibilities:  Provide support to Director and act as the primary liaison with various divisions/department or any newly set-up branch within...                                                                         https://www.jobstreet.com.my/en/job/personal-assistant-secretary-to-director-4980388?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=8&jobId=jobstreet-my-job-4980388    2000-4000 MYR  Kuala Lumpur, Selangor
8                                                           Personal Assistant          Personal Assistants perform a variety of administrative tasks for an individual.Here are some common responsibilities of a Personal Assistant:...                                                                                               https://www.jobstreet.com.my/en/job/personal-assistant-4986053?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=9&jobId=jobstreet-my-job-4986053    2000-4000 MYR             Subang Jaya
9                          Special Officer to CEO - Public Listed Conglomerate  Reporting and supporting the CEO in his daily work in wide variety of area - including preparation for meetings, development of presentation materials...                                                               https://www.jobstreet.com.my/en/job/special-officer-to-ceo-public-listed-conglomerate-4986807?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=10&jobId=jobstreet-my-job-4986807  15000-20000 MYR            Kuala Lumpur
...

轉載請註明出處，本文鏈接：https://www.uj5u.com/ruanti/490636.html

標籤：Python 网页抓取

上一篇：PythonSelenium：使用findElementByXPath回傳的表元素使用id欄位回傳空字串

下一篇：如何用scrapy提取Json