如何抓取具有相同类的另一个span

Question

我使用的是beautifulSoup4，这是我的代码:

def extract(page):
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/{page}/'
r = requests.get(url)
soup = BeautifulSoup(r.content,'html.parser')
return soup
def transform(soup):
divs = soup.find_all('div', class_ = 'sx2jih0 zcydq876 zcydq866 zcydq896 zcydq886 zcydq8n zcydq856 zcydq8f6 zcydq8eu')
for items in divs:
location = items.find('span', attrs={'class': 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'}).text.strip()
salary = items.find_next_sibling('span', attrs={'class': 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'}).text.strip()

两个span都有相同的类，但是当我取消时，两个结果是相同的。

Answer 1

通过find_all获取所有的span，然后删除那些小于2个span的项目。然后从span列表中获取location和salary

import requests
from bs4 import BeautifulSoup
def extract(page):
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/{page}/'
r = requests.get(url)
soup = BeautifulSoup(r.content,'html.parser')
# transform
jobs = soup.find_all('div', class_ = 'sx2jih0 zcydq876 zcydq866 zcydq896 zcydq886 zcydq8n zcydq856 zcydq8f6 zcydq8eu')
data = []
for job in jobs[:100]:
items = job.find_all('span', attrs = {'class' : 'sx2jih0 zcydq84u _18qlyvc0 _18qlyvc1x _18qlyvc3 _18qlyvc7'})
if len(items) < 2:
continue
location = items[0].text.strip()
salary = items[1].text.strip()
data.append([location, salary])

return data

data = extract(1)
print(data)

Answer 2

尽量避免选择HTML中的动态部分，如类-当您仍然找出兄弟关系时，您可以使用find_next_sibling('span')。

下面将只按标签选择，并检查是否有工资，以避免错误而刮掉整个工作，同时显示工资None:

for item in soup.select('article'):
data.append({
'title': item.h1.text,
'location': item.h1.parent.next_sibling.text,
'salary': item.h1.parent.next_sibling.find_next_sibling('span').text if item.h1.parent.next_sibling.find_next_sibling('span') else None,
})

import requests
url = f'https://www.jobstreet.com.my/en/job-search/personal-assistant-jobs/1/'
r = requests.get(url)
soup = BeautifulSoup(r.content, 'html.parser')
data=[]
for item in soup.select('article'):
data.append({
'title': item.h1.text,
'location': item.h1.parent.next_sibling.text,
'salary': item.h1.parent.next_sibling.find_next_sibling('span').text if item.h1.parent.next_sibling.find_next_sibling('span') else None,
})
data

Answer 3

使用动态类并不是最好的主意。该网站使用api来搜索职位和图表。

import requests
import json
import pandas as pd

def foo(page, keyword):
url = "https://xapi.supercharge-srp.co/job-search/graphql?country=my&isSmartSearch=true"
payload = json.dumps({
"query": "query getJobs($country: String, $locale: String, $keyword: String, $createdAt: String, $jobFunctions: [Int], $categories: [String], $locations: [Int], $careerLevels: [Int], $minSalary: Int, $maxSalary: Int, $salaryType: Int, $candidateSalary: Int, $candidateSalaryCurrency: String, $datePosted: Int, $jobTypes: [Int], $workTypes: [String], $industries: [Int], $page: Int, $pageSize: Int, $companyId: String, $advertiserId: String, $userAgent: String, $accNums: Int, $subAccount: Int, $minEdu: Int, $maxEdu: Int, $edus: [Int], $minExp: Int, $maxExp: Int, $seo: String, $searchFields: String, $candidateId: ID, $isDesktop: Boolean, $isCompanySearch: Boolean, $sort: String, $sVi: String, $duplicates: String, $flight: String, $solVisitorId: String) {n  jobs(n    country: $countryn    locale: $localen    keyword: $keywordn    createdAt: $createdAtn    jobFunctions: $jobFunctionsn    categories: $categoriesn    locations: $locationsn    careerLevels: $careerLevelsn    minSalary: $minSalaryn    maxSalary: $maxSalaryn    salaryType: $salaryTypen    candidateSalary: $candidateSalaryn    candidateSalaryCurrency: $candidateSalaryCurrencyn    datePosted: $datePostedn    jobTypes: $jobTypesn    workTypes: $workTypesn    industries: $industriesn    page: $pagen    pageSize: $pageSizen    companyId: $companyIdn    advertiserId: $advertiserIdn    userAgent: $userAgentn    accNums: $accNumsn    subAccount: $subAccountn    minEdu: $minEdun    edus: $edusn    maxEdu: $maxEdun    minExp: $minExpn    maxExp: $maxExpn    seo: $seon    searchFields: $searchFieldsn    candidateId: $candidateIdn    isDesktop: $isDesktopn    isCompanySearch: $isCompanySearchn    sort: $sortn    sVi: $sVin    duplicates: $duplicatesn    flight: $flightn    solVisitorId: $solVisitorIdn  ) {n    totaln    totalJobsn    relatedSearchKeywords {n      keywordsn      typen      totalJobsn    }n    solMetadatan    suggestedEmployer {n      namen      totalJobsn    }n    queryParameters {n      keyn      searchFieldsn      pageSizen    }n    experiments {n      flightn    }n    jobs {n      idn      adTypen      sourceCountryCoden      isStandoutn      companyMeta {n        idn        advertiserIdn        isPrivaten        namen        logoUrln        slugn      }n      jobTitlen      jobUrln      jobTitleSlugn      descriptionn      employmentTypes {n        coden        namen      }n      sellingPointsn      locations {n        coden        namen        slugn        children {n          coden          namen          slugn        }n      }n      categories {n        coden        namen        children {n          coden          namen        }n      }n      postingDurationn      postedAtn      salaryRange {n        currencyn        maxn        minn        periodn        termn      }n      salaryVisiblen      bannerUrln      isClassifiedn      solMetadatan    }n  }n}n",
"variables": {
"keyword": keyword,
"jobFunctions": [],
"locations": [],
"salaryType": 1,
"jobTypes": [],
"createdAt": None,
"careerLevels": [],
"page": page,
"country": "my",
"categories": [],
"workTypes": [],
"userAgent": "Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36",
"industries": [],
"locale": "en"
}
})
headers = {
'accept': '*/*',
'content-type': 'application/json'
}
response = requests.request("POST", url, headers=headers, data=payload)
salary = "Not specified"
result = []
for job in (response.json()['data']['jobs']['jobs']):
locations = []
if job['salaryVisible']:
salary = f"{job['salaryRange']['min']}-{job['salaryRange']['max']} {job['salaryRange']['currency']}"
for location in job['locations']:
locations.append(location['name'])
result.append([job['jobTitle'], job['description'], job['jobUrl'], salary, ", ".join(locations)])
df = pd.DataFrame(result)
df.columns = ['Title', 'Description', 'Link', 'Salary', 'Location']
print(df.to_string())

foo(1, "personal assistant")

输出:

   Title                                                                                                                                                Description                                                                                                                                                                                                                                                   Link           Salary                Location
0                                                Personal Assistant/ Secretary        Job Description:GETO GLOBAL CONSTRUCTION TECH MALAYSIA SDN. BHD. is a subsidiary of a public listed company from China, specializing in Aluminum...                                                                                     https://www.jobstreet.com.my/en/job/personal-assistant-secretary-4986870?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=1&jobId=jobstreet-my-job-4986870    3000-6000 MYR        Shah Alam/Subang
1                                                          Executive Assistant      Responsibilities:  Responsible in managing day-to-day office related and administrative tasks. Organizing and attending meetings and ensuring well...                                                                                              https://www.jobstreet.com.my/en/job/executive-assistant-4986449?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=2&jobId=jobstreet-my-job-4986449    2000-3000 MYR          Johor - Others
2                                                                    Secretary        A good write-up Secretary's resume is a Must with experience and can handle Managing Director company matters only. Fresh grads or many years in...                                                                                                        https://www.jobstreet.com.my/en/job/secretary-4986714?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=3&jobId=jobstreet-my-job-4986714    2000-3000 MYR           Petaling Jaya
3                                                 Secretary/Personal assistant  We are looking for Secretary/Personal assistant in our company for: Organising and maintaining office systems Screening phone calls, emails, enquiries...                                                                                     https://www.jobstreet.com.my/en/job/secretary-personal-assistant-4983131?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=4&jobId=jobstreet-my-job-4983131    2000-3000 MYR            Kuala Lumpur
4                                         Personal Assistant to Country Leader    Do you want to make a positive impact on the world? Sustainable development is at the heart of everything we do.Seventy-five years ago, we created a...                                                                             https://www.jobstreet.com.my/en/job/personal-assistant-to-country-leader-4986906?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=5&jobId=jobstreet-my-job-4986906    2000-3000 MYR           Petaling Jaya
5                                        Personal Assistant to Senior Director  Requirement: 5 – 8 years in handling PA end to end tasks for a Director/CEO/Managing Director Highly proficient in MS Office software Strong knowledge...                                                                            https://www.jobstreet.com.my/en/job/personal-assistant-to-senior-director-4986792?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=6&jobId=jobstreet-my-job-4986792    2000-3000 MYR        Shah Alam/Subang
6                                                           Personal Assistant     Report to top management; Attend to all aspects of PA and corporate work; Draft and manage correspondences, email, and memos; Schedule and organize...                                                                                               https://www.jobstreet.com.my/en/job/personal-assistant-4987086?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=7&jobId=jobstreet-my-job-4987086    2000-4000 MYR                 Puchong
7                                   Personal Assistant (Secretary) to Director       Responsibilities:  Provide support to Director and act as the primary liaison with various divisions/department or any newly set-up branch within...                                                                         https://www.jobstreet.com.my/en/job/personal-assistant-secretary-to-director-4980388?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=8&jobId=jobstreet-my-job-4980388    2000-4000 MYR  Kuala Lumpur, Selangor
8                                                           Personal Assistant          Personal Assistants perform a variety of administrative tasks for an individual.Here are some common responsibilities of a Personal Assistant:...                                                                                               https://www.jobstreet.com.my/en/job/personal-assistant-4986053?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=9&jobId=jobstreet-my-job-4986053    2000-4000 MYR             Subang Jaya
9                          Special Officer to CEO - Public Listed Conglomerate  Reporting and supporting the CEO in his daily work in wide variety of area - including preparation for meetings, development of presentation materials...                                                               https://www.jobstreet.com.my/en/job/special-officer-to-ceo-public-listed-conglomerate-4986807?token=0~5e9badd8-c329-4c4c-b157-bc43eaa5cd24&sectionRank=10&jobId=jobstreet-my-job-4986807  15000-20000 MYR            Kuala Lumpur
...