webscraperbeautifulsouppython中的不同標簽

我試圖抓取一個股票網站抓取行業和行業（下一個問題）并將其添加到 csv 中。我得到了我想要的 1 頁的資訊，但下一個是不同的，所以這就是我卡住的地方

share_details1 = soup.find('a', href="../Industry/Industry_Data.php?s=100") 結果：基本材料我想找到范圍從 100 到 1300 的標簽，例如 href="。 ./Industry/Industry_Data.php?s=200" 300 400 500 600 依此類推至 1300

from bs4 import BeautifulSoup as bs
import csv
import requests

LSE = 'https://csimarket.com/stocks/at_glance.php?code=aa'

def get_stocks():
    with open('tickers.csv') as ticker_file:
        return list(map(lambda ticker: ticker.strip(), ticker_file))


def to_csv(stocks):
    with open('stocks.csv', 'w') as sectors:
        writer = csv.writer(sectors)
        writer.writerow(stocks[0].keys())
        for stock in stocks:
            writer.writerow(stock.values())


def get_soup(url):
    return bs(requests.get(url).text, 'html.parser')

def get_sector(ticker):
    soup = get_soup(LSE   ticker)
    try:
        share_details1 = soup.find('a', href="../Industry/Industry_Data.php?s=100")
        messy = share_details1.find("span")
        messy.decompose()
        sector = share_details1.text.strip()

    except:
        print('No sector information availible for ', ticker)
        return {'ticker': ticker, 'sector': ''}

    print(ticker, sector)
    return {'ticker': ticker, 'sector': sector}


def get_industry(ticker):
    soup1 = get_soup(LSE   ticker)
    try:
        share_details1 = soup1.find('a', href="../Industry/Industry_Data.php?ind=104")
        messy = share_details1.find("span")
        messy.decompose()
        industry = share_details1.text.strip()

    except:
        print('No industry information availible for ', ticker)
        return {'ticker': ticker, 'industry': ''}

    print(ticker, industry)
    return {'ticker': ticker, 'industry': industry}


if __name__ == '__main__':
    to_csv(list(map(lambda ticker: get_sector(ticker), get_stocks())))
    # to_csv(list(map(lambda ticker: get_industry(ticker), get_stocks())))

這是csv的樣本

ticker,sector
A,
AA,Basic Materials
AADI,
AAIC,
AAL,
AAN,
AAOI,
AAON,
AAP,
AAPL,
AAT,
AAU,Basic Materials
AAWW,
AB,
ABB,
ABBV,
ABC,
ABCB,
ABCL,
ABEO,
ABEV,
ABG,
ABIO,
ABM,
ABMD,
ABNB,
ABOS,
ABR,
ABSI,
ABST,
ABT,
ABTX,
ABUS,
ACA,Basic Materials
ACAD,
ACB,
ACC,
ACCD,
ACCO,Basic Materials
ACEL,
ACER,
ACET,
ACEV,
ACGL,
ACH,Basic Materials
ACHC,
ACHR,
ACHV,
ACI,
ACIU,

uj5u.com熱心網友回復：

看起來那些hrefs是動態的。你最好只尋找'Sector'or'Industry'然后決議它。

您也可以使用正則運算式提取該資訊。但這是我的解決方法。

from bs4 import BeautifulSoup as bs
import pandas as pd
import re
import requests

LSE = 'https://csimarket.com/stocks/at_glance.php?code='

def get_stocks():
    df = pd.read_csv('tickers.csv')
    return list(df['ticker'])


def to_csv(stocks):
    df = pd.DataFrame(stocks)
    df.to_csv('stocks.csv', index=False)

def get_soup(url):
    return bs(requests.get(url).text, 'html.parser')

def get_sector(ticker):
    soup = get_soup(LSE   ticker)
    try:
        sector = soup.find('span', text='Sector').find_next('a').text.replace('\n','').replace('?','').strip()
    except:
        print('No sector information availible for ', ticker)
        return {'ticker': ticker, 'sector': ''}

    print(ticker, sector)
    return {'ticker': ticker, 'sector': sector}


def get_industry(ticker):
    soup1 = get_soup(LSE   ticker)
    try:
        industry = soup1.find('span', text='Industry').find_next('a').text.replace('\n','').replace('?','').strip()
    except:
        print('No industry information availible for ', ticker)
        return {'ticker': ticker, 'industry': ''}

    print(ticker, industry)
    return {'ticker': ticker, 'industry': industry}


if __name__ == '__main__':
    to_csv(list(map(lambda ticker: get_sector(ticker), get_stocks())))
    # to_csv(list(map(lambda ticker: get_industry(ticker), get_stocks())))

標籤：Python 网页抓取 美丽的汤 css 选择器 标签

上一篇：如何使用Selenium獲取各州不同餐廳的選單項和價格？

下一篇：如何在錯誤物件中列印屬性值？