我正在嘗試動態創建 python 字典鍵，以便將資料提供到 csv 檔案中，但到目前為止還沒有到達任何地方。這是我的代碼：
import requests
from bs4 import BeautifulSoup
import csv
class ZiwiScraper:
results = []
headers = {
'authority': '99petshops.com.au',
'accept': 'text/html,application/xhtml xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-language': 'en,ru;q=0.9',
'cache-control': 'max-age=0',
# Requests sorts cookies= alphabetically
# 'cookie': 'TrackerGuid=f5419f8d-632a-46b1-aa04-eed027d03e89; _ga=GA1.3.1385392550.1666770065; _gid=GA1.3.1560927430.1666770065',
'referer': 'https://www.upwork.com/',
'sec-ch-ua': '"Chromium";v="104", " Not A;Brand";v="99", "Yandex";v="22"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Linux"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'cross-site',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.114 YaBrowser/22.9.1.1110 (beta) Yowser/2.5 Safari/537.36',
}
def fetch(self, url):
print(f'HTTP GET request to URL: {url}', end='')
res = requests.get(url, headers=self.headers)
print(f' | Status Code: {res.status_code}')
return res
def parse(self, html):
soup = BeautifulSoup(html, 'lxml')
titles = [title.text.strip() for title in soup.find_all('h2')]
low_prices = [low_price.text.split(' ')[-1] for low_price in soup.find_all('span', {'class': 'hilighted'})]
store_names = []
stores = soup.find_all('p')
for store in stores:
store_name = store.find('img')
if store_name:
store_names.append(store_name['alt'])
shipping_prices = [shipping.text.strip() for shipping in soup.find_all('p', {'class': 'shipping'})]
price_per_hundered_kg = [unit_per_kg.text.strip() for unit_per_kg in soup.find_all('p', {'class': 'unit-price'})]
other_details = soup.find_all('div', {'class': 'pd-details'})
for index in range(0, len(titles)):
try:
price_per_100_kg = price_per_hundered_kg[index]
except:
price_per_100_kg = ''
try:
lowest_prices = low_prices[index]
except:
lowest_prices = ''
for detail in other_details:
detail_1 = [pr.text.strip() for pr in detail.find_all('span', {'class': 'sp-price'})]
for idx, price in enumerate(detail_1):
self.results.append({
'title': titles[index],
'lowest_prices': lowest_prices,
f'lowest_price_{idx}': detail_1[idx],
'store_names': store_names[index],
'shipping_prices': shipping_prices[index],
'price_per_100_kg': price_per_100_kg,
})
# json_object = json.dumps(self.results, indent=4)
# with open("ziwi_pets_2.json", "w") as outfile:
# outfile.write(json_object)
def to_csv(self):
with open('ziwi_pets_2.csv', 'w') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=self.results[0].keys())
writer.writeheader()
for row in self.results:
writer.writerow(row)
print('Stored results to "ziwi_pets_2.csv"')
def run(self):
for page in range(1):
url = f'https://99petshops.com.au/Search?brandName=Ziwi Peak&animalCode=DOG&storeId=89/&page={page}'
response = self.fetch(url)
self.parse(response.text)
self.to_csv()
if __name__ == '__main__':
scraper = ZiwiScraper()
scraper.run()
每次我運行腳本時，它都會給我上面的代碼ValueError: dict contains fields not in fieldnames: 'lowest_price_1'。csv 檔案，但僅生成一個條目。
title,lowest_prices,lowest_price_0,store_names,shipping_prices,price_per_100_kg
Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 1Kg,$57.75,$64.60,Woofers World, $9.95 shipping,$5.78 per 100g
我試圖將它輸出為 json 只是為了查看資料形成，它也不是我預期的那樣。
[
{
"title": "Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 1Kg",
"lowest_prices": "$57.75",
"lowest_price_0": "$64.60",
"store_names": "Woofers World",
"shipping_prices": " $9.95 shipping",
"price_per_100_kg": "$5.78 per 100g"
},
{
"title": "Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 1Kg",
"lowest_prices": "$57.75",
"lowest_price_1": "$64.60",
"store_names": "Woofers World",
"shipping_prices": " $9.95 shipping",
"price_per_100_kg": "$5.78 per 100g"
},
{
"title": "Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 1Kg",
"lowest_prices": "$57.75",
"lowest_price_2": "$64.95",
"store_names": "Woofers World",
"shipping_prices": " $9.95 shipping",
"price_per_100_kg": "$5.78 per 100g"
},
]
我期待這樣的事情：
[
{
"title": "Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 1Kg",
"lowest_prices": "$57.75",
"lowest_price_0": "$64.60",
"lowest_price_1": "$64.60",
"lowest_price_2": "$64.95",
"store_names": "Woofers World",
"shipping_prices": " $9.95 shipping",
"price_per_100_kg": "$5.78 per 100g"
},
]
任何人都可以在這里幫助我嗎？謝謝。
uj5u.com熱心網友回復：
您需要傳遞任何專案可能具有的fieldnames 所有鍵。self.results如果您只是將其添加到您的to_csv函式中，該錯誤應該會消失
def to_csv(self):
fNames = list(set([k for r in self.results for k in r.keys()]))
fNames.sort(key=lambda r: (
-1*(int(r.replace('lowest_price_', '')) 1) if 'lowest_price_' in r else 0, r
), reverse=True)
with open('ziwi_pets_2.csv', 'w') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=fNames)
writer.writeheader()
for row in self.results:
writer.writerow(row)
print('Stored results to "ziwi_pets_2.csv"')
（該sort陳述句是可選的 - 它只是為了確保列按特定順序。）
至于這個位
我期待這樣的事情
for idx, price in enumerate(detail_1)函式中的為 each的每個最低價格 IDx 值parse附加一個新行。要將它們全部放在一行中，請將外部回圈更改為detail
for detail in other_details:
result_i = {
'title': titles[index],
'lowest_prices': lowest_prices,
'store_names': store_names[index],
'shipping_prices': shipping_prices[index],
'price_per_100_kg': price_per_100_kg,
}
detail_1 = [pr.text.strip() for pr in detail.find_all('span', {'class': 'sp-price'})]
for idx, price in enumerate(detail_1):
result_i[f'lowest_price_{idx}'] = detail_1[idx]
self.results.append(result_i)
然后，您的 csv 輸出將從
到
你的json會看起來像預期的那樣
{
"title": "Ziwi Peak Dog Air-Dried Free Range Chicken Recipe 1Kg",
"lowest_prices": "$57.75",
"store_names": "Woofers World",
"shipping_prices": " $9.95 shipping",
"price_per_100_kg": "$5.78 per 100g",
"lowest_price_0": "$3.87",
"lowest_price_1": "$4.49",
"lowest_price_2": "$5.09",
"lowest_price_3": "$5.95",
"lowest_price_4": "$5.99"
}
（使用列印print(json.dumps([r for r in scraper.results if len(r) == 10][0], indent=4))）
uj5u.com熱心網友回復：
您僅根據第一個結果創建 csv 標頭，該結果不包含完整的唯一鍵集。
將 to_csv 函式更改為此以解決您的問題：
def to_csv(self):
key_list = list()
for key_list_for_one_element in [list(x.keys()) for x in self.results]:
key_list.extend(key_list_for_one_element)
key_list = set(key_list)
with open('ziwi_pets_2.csv', 'w') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=key_list)
writer.writeheader()
for row in self.results:
writer.writerow(row)
print('Stored results to "ziwi_pets_2.csv"')
轉載請註明出處，本文鏈接：https://www.uj5u.com/yidong/523239.html