Das deutsche Python-Forum

Code: Alles auswählen

['MMM\n', 'ABT\n', 'ABBV\n', 'ABMD\n', 'ACN\n', 'ATVI\n', 'ADBE\n', 'AMD\n', 'AAP\n', 'AES\n', 'AMG\n', 'AFL\n', 'A\n', 'APD\n', 'AKAM\n', 'ALK\n', 'ALB\n', 'ARE\n', 'ALXN\n', 'ALGN\n', 'ALLE\n', 'AGN\n', 'ADS\n', 'LNT\n', 'ALL\n', 'GOOGL\n', 'GOOG\n', 'MO\n', 'AMZN\n', 'AMCR\n', 'AEE\n', 'AAL\n', 'AEP\n', 'AXP\n', 'AIG\n', 'AMT\n', 'AWK\n', 'AMP\n', 'ABC\n', 'AME\n', 'AMGN\n', 'APH\n', 'ADI\n', 'ANSS\n', 'ANTM\n', 'AON\n', 'AOS\n', 'APA\n', 'AIV\n', 'AAPL\n', 'AMAT\n', 'APTV\n', 'ADM\n', 'ARNC\n', 'ANET\n', 'AJG\n', 'AIZ\n', 'ATO\n', 'T\n', 'ADSK\n', 'ADP\n', 'AZO\n', 'AVB\n', 'AVY\n', 'BHGE\n', 'BLL\n', 'BAC\n', 'BK\n', 'BAX\n', 'BBT\n', 'BDX\n', 'BRK.B\n', 'BBY\n', 'BIIB\n', 'BLK\n', 'HRB\n', 'BA\n', 'BKNG\n', 'BWA\n', 'BXP\n', 'BSX\n', 'BMY\n', 'AVGO\n', 'BR\n', 'BF.B\n', 'CHRW\n', 'COG\n', 'CDNS\n', 'CPB\n', 'COF\n', 'CPRI\n', 'CAH\n', 'KMX\n', 'CCL\n', 'CAT\n', 'CBOE\n', 'CBRE\n', 'CBS\n', 'CE\n', 'CELG\n', 'CNC\n', 'CNP\n', 'CTL\n', 'CERN\n', 'CF\n', 'SCHW\n', 'CHTR\n', 'CVX\n', 'CMG\n', 'CB\n', 'CHD\n', 'CI\n', 'XEC\n', 'CINF\n', 'CTAS\n', 'CSCO\n', 'C\n', 'CFG\n', 'CTXS\n', 'CLX\n', 'CME\n', 'CMS\n', 'KO\n', 'CTSH\n', 'CL\n', 'CMCSA\n', 'CMA\n', 'CAG\n', 'CXO\n', 'COP\n', 'ED\n', 'STZ\n', 'COO\n', 'CPRT\n', 'GLW\n', 'CTVA\n', 'COST\n', 'COTY\n', 'CCI\n', 'CSX\n', 'CMI\n', 'CVS\n', 'DHI\n', 'DHR\n', 'DRI\n', 'DVA\n', 'DE\n', 'DAL\n', 'XRAY\n', 'DVN\n', 'FANG\n', 'DLR\n', 'DFS\n', 'DISCA\n', 'DISCK\n', 'DISH\n', 'DG\n', 'DLTR\n', 'D\n', 'DOV\n', 'DOW\n', 'DTE\n', 'DUK\n', 'DRE\n', 'DD\n', 'DXC\n', 'ETFC\n', 'EMN\n', 'ETN\n', 'EBAY\n', 'ECL\n', 'EIX\n', 'EW\n', 'EA\n', 'EMR\n', 'ETR\n', 'EOG\n', 'EFX\n', 'EQIX\n', 'EQR\n', 'ESS\n', 'EL\n', 'EVRG\n', 'ES\n', 'RE\n', 'EXC\n', 'EXPE\n', 'EXPD\n', 'EXR\n', 'XOM\n', 'FFIV\n', 'FB\n', 'FAST\n', 'FRT\n', 'FDX\n', 'FIS\n', 'FITB\n', 'FE\n', 'FRC\n', 'FISV\n', 'FLT\n', 'FLIR\n', 'FLS\n', 'FMC\n', 'F\n', 'FTNT\n', 'FTV\n', 'FBHS\n', 'FOXA\n', 'FOX\n', 'BEN\n', 'FCX\n', 'GPS\n', 'GRMN\n', 'IT\n', 'GD\n', 'GE\n', 'GIS\n', 'GM\n', 'GPC\n', 'GILD\n', 'GL\n', 'GPN\n', 'GS\n', 'GWW\n', 'HAL\n', 'HBI\n', 'HOG\n', 'HIG\n', 'HAS\n', 'HCA\n', 'HCP\n', 'HP\n', 'HSIC\n', 'HSY\n', 'HES\n', 'HPE\n', 'HLT\n', 'HFC\n', 'HOLX\n', 'HD\n', 'HON\n', 'HRL\n', 'HST\n', 'HPQ\n', 'HUM\n', 'HBAN\n', 'HII\n', 'IEX\n', 'IDXX\n', 'INFO\n', 'ITW\n', 'ILMN\n', 'IR\n', 'INTC\n', 'ICE\n', 'IBM\n', 'INCY\n', 'IP\n', 'IPG\n', 'IFF\n', 'INTU\n', 'ISRG\n', 'IVZ\n', 'IPGP\n', 'IQV\n', 'IRM\n', 'JKHY\n', 'JEC\n', 'JBHT\n', 'JEF\n', 'SJM\n', 'JNJ\n', 'JCI\n', 'JPM\n', 'JNPR\n', 'KSU\n', 'K\n', 'KEY\n', 'KEYS\n', 'KMB\n', 'KIM\n', 'KMI\n', 'KLAC\n', 'KSS\n', 'KHC\n', 'KR\n', 'LB\n', 'LHX\n', 'LH\n', 'LRCX\n', 'LW\n', 'LEG\n', 'LDOS\n', 'LEN\n', 'LLY\n', 'LNC\n', 'LIN\n', 'LKQ\n', 'LMT\n', 'L\n', 'LOW\n', 'LYB\n', 'MTB\n', 'MAC\n', 'M\n', 'MRO\n', 'MPC\n', 'MKTX\n', 'MAR\n',
'MMC\n', 'MLM\n', 'MAS\n', 'MA\n', 'MKC\n', 'MXIM\n', 'MCD\n', 'MCK\n', 'MDT\n', 'MRK\n', 'MET\n', 'MTD\n', 'MGM\n', 'MCHP\n', 'MU\n', 'MSFT\n', 'MAA\n', 'MHK\n', 'TAP\n', 'MDLZ\n', 'MNST\n', 'MCO\n', 'MS\n', 'MOS\n', 'MSI\n', 'MSCI\n', 'MYL\n', 'NDAQ\n', 'NOV\n', 'NKTR\n', 'NTAP\n', 'NFLX\n', 'NWL\n', 'NEM\n', 'NWSA\n', 'NWS\n', 'NEE\n', 'NLSN\n', 'NKE\n', 'NI\n', 'NBL\n', 'JWN\n', 'NSC\n', 'NTRS\n', 'NOC\n', 'NCLH\n', 'NRG\n', 'NUE\n', 'NVDA\n', 'ORLY\n', 'OXY\n', 'OMC\n', 'OKE\n', 'ORCL\n', 'PCAR\n', 'PKG\n', 'PH\n', 'PAYX\n', 'PYPL\n', 'PNR\n', 'PBCT\n', 'PEP\n', 'PKI\n', 'PRGO\n', 'PFE\n', 'PM\n', 'PSX\n', 'PNW\n', 'PXD\n', 'PNC\n', 'PPG\n', 'PPL\n', 'PFG\n', 'PG\n', 'PGR\n', 'PLD\n', 'PRU\n',
'PEG\n', 'PSA\n', 'PHM\n', 'PVH\n', 'QRVO\n', 'PWR\n', 'QCOM\n', 'DGX\n', 'RL\n', 'RJF\n', 'RTN\n', 'O\n', 'REG\n', 'REGN\n', 'RF\n', 'RSG\n', 'RMD\n', 'RHI\n', 'ROK\n', 'ROL\n', 'ROP\n', 'ROST\n', 'RCL\n', 'CRM\n', 'SBAC\n', 'SLB\n', 'STX\n', 'SEE\n', 'SRE\n', 'SHW\n', 'SPG\n', 'SWKS\n', 'SLG\n', 'SNA\n', 'SO\n', 'LUV\n', 'SPGI\n', 'SWK\n', 'SBUX\n',
'STT\n', 'SYK\n', 'STI\n', 'SIVB\n', 'SYMC\n', 'SYF\n', 'SNPS\n', 'SYY\n', 'TMUS\n', 'TROW\n', 'TTWO\n', 'TPR\n', 'TGT\n', 'TEL\n', 'FTI\n', 'TFX\n', 'TXN\n', 'TXT\n', 'TMO\n',
'TIF\n', 'TWTR\n', 'TJX\n', 'TSS\n', 'TSCO\n', 'TDG\n', 'TRV\n', 'TRIP\n', 'TSN\n', 'UDR\n', 'ULTA\n', 'USB\n', 'UAA\n', 'UA\n', 'UNP\n', 'UAL\n', 'UNH\n', 'UPS\n', 'URI\n', 'UTX\n', 'UHS\n', 'UNM\n', 'VFC\n', 'VLO\n', 'VAR\n', 'VTR\n', 'VRSN\n', 'VRSK\n', 'VZ\n', 'VRTX\n', 'VIAB\n', 'V\n', 'VNO\n', 'VMC\n', 'WAB\n', 'WMT\n', 'WBA\n', 'DIS\n', 'WM\n',
'WAT\n', 'WEC\n', 'WCG\n', 'WFC\n', 'WELL\n', 'WDC\n', 'WU\n', 'WRK\n', 'WY\n', 'WHR\n', 'WMB\n', 'WLTW\n', 'WYNN\n', 'XEL\n', 'XRX\n', 'XLNX\n', 'XYL\n', 'YUM\n', 'ZBH\n', 'ZION\n', 'ZTS\n']

Code: Alles auswählen

def save_sp500_tickers():
    resp = requests.get('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
    soup = bs.BeautifulSoup(resp.text, features='lxml')
    table = soup.find('table', {'class': 'wikitable sortable'})
    tickers = []
    for row in table.findAll('tr')[1:]:
        ticker = row.findAll('td')[0].text[0:4]
        if ticker[2] == "\\":
            tickers.append(ticker[0:1])
        elif ticker[3] == "\\":
            tickers.append(ticker[0:2])
        else:
            tickers.append(ticker[0:3])

    with open('sp500tickers.pickle', 'wb') as f:
        pickle.dump(tickers, f)

    print(tickers)

    return tickers

Code: Alles auswählen

#!/usr/bin/env python3
import json

import bs4
import requests


def retrieve_sp500_tickers():
    response = requests.get(
        "https://en.wikipedia.org/wiki/List_of_S&P_500_companies"
    )
    response.raise_for_status()
    soup = bs4.BeautifulSoup(response.text, "lxml")
    first_row = soup.find("table", "wikitable sortable").tr
    if first_row.th.text.strip() != "Symbol":
        raise ValueError(
            f"Unexcpected header value {first_row.th.text.strip()}"
        )
    tickers = [
        row.td.text.strip() for row in first_row.find_next_siblings("tr")
    ]

    with open("sp500tickers.json", "w", encoding="ascii") as file:
        json.dump(tickers, file)


if __name__ == "__main__":
    retrieve_sp500_tickers()

Code: Alles auswählen

def get_data_from_yahoo(reload_sp500=False):
    if reload_sp500:
        tickers = save_sp500_tickers()
    else:
        with open('sp500tickers.pickle', 'rb') as f:
            tickers = pickle.load(f)

    if not os.path.exists('stock_dfs'):
        os.makedirs('stock_dfs')

    start = dt.datetime(2000, 1, 1)
    end = dt.datetime(2016, 12, 31)

    for ticker in tickers[:5]:
        print(ticker)
        if not os.path.exists('stock_dfs/{}.csv'.format(ticker)):
            df = web.DataReader(ticker, 'yahoo', start, end)
            df.to_csv('stock_dfs/{}.csv'.format(ticker))
        else:
            print('Already have {}'.format(ticker))

Code: Alles auswählen

def retrieve_sp500_tickers():
    response = requests.get(
        'https://en.wikipedia.org/wiki/List_of_S&P_500_companies'
    )
    response.raise_for_status()
    soup = bs.BeautifulSoup(response.text, 'lxml')
    first_row = soup.find('table', 'wikitable sortable').tr
    if first_row.th.text.strip() != 'Symbol':
        raise ValueError(
            f'Unexcpected header value {first_row.th.text.strip()}'
        )
    tickers = [
        row.td.text.strip() for row in first_row.find_next_siblings('tr')
    ]

    with open('sp500tickers.json', 'w', encoding='ascii') as file:
        json.dump(tickers, file)

    print(tickers)

    return tickers

def get_data_from_yahoo(reload_sp500=False):
    if reload_sp500:
        tickers = retrieve_sp500_tickers()
    else:
        with open('sp500tickers.json', 'rb') as f:
            tickers = json.load(f)

    if not os.path.exists('stock_dfs'):
        os.makedirs('stock_dfs')

    start = dt.datetime(2000, 1, 1)
    end = dt.datetime(2016, 12, 31)

    for ticker in tickers:
        print(ticker)
        if '.' in ticker:
            ticker = ticker.replace('.', '')
            print('Dot removed from variable - {}'.format(ticker))
        if not os.path.exists('stock_dfs/{}.csv'.format(ticker)):
            df = web.DataReader(ticker, 'yahoo', start, end)
            print(df)
            df.to_csv('stock_dfs/{}.csv'.format(ticker))
        else:
            print('Already have {}'.format(ticker))
        
get_data_from_yahoo()

Code: Alles auswählen

python main.py
['MMM', 'ABT', 'ABBV', 'ABMD', 'ACN', 'ATVI', 'ADBE', 'AMD', 'AAP', 'AES', 'AMG', 'AFL', 'A', 'APD', 'AKAM', 'ALK', 'ALB', 'ARE', 'ALXN', 'ALGN', 'ALLE', 'AGN', 'ADS', 'LNT', 'ALL', 'GOOGL', 'GOOG', 'MO', 'AMZN', 'AMCR', 'AEE', 'AAL', 'AEP', 'AXP', 'AIG', 'AMT', 'AWK', 'AMP', 'ABC', 'AME', 'AMGN', 'APH', 'ADI', 'ANSS', 'ANTM', 'AON', 'AOS', 'APA', 'AIV', 'AAPL', 'AMAT', 'APTV', 'ADM', 'ARNC', 'ANET', 'AJG', 'AIZ', 'ATO', 'T', 'ADSK', 'ADP', 'AZO', 'AVB', 'AVY', 'BHGE', 'BLL', 'BAC', 'BK', 'BAX', 'BBT', 'BDX', 'BRK.B', 'BBY',
'BIIB', 'BLK', 'HRB', 'BA', 'BKNG', 'BWA', 'BXP', 'BSX', 'BMY', 'AVGO', 'BR', 'BF.B', 'CHRW', 'COG', 'CDNS', 'CPB', 'COF', 'CPRI', 'CAH', 'KMX', 'CCL', 'CAT', 'CBOE', 'CBRE', 'CBS', 'CE', 'CELG', 'CNC', 'CNP', 'CTL', 'CERN', 'CF', 'SCHW', 'CHTR', 'CVX', 'CMG', 'CB', 'CHD', 'CI', 'XEC', 'CINF', 'CTAS', 'CSCO', 'C', 'CFG', 'CTXS', 'CLX', 'CME', 'CMS', 'KO', 'CTSH', 'CL', 'CMCSA', 'CMA', 'CAG', 'CXO', 'COP', 'ED', 'STZ', 'COO', 'CPRT', 'GLW', 'CTVA', 'COST', 'COTY', 'CCI', 'CSX', 'CMI', 'CVS', 'DHI', 'DHR', 'DRI', 'DVA', 'DE', 'DAL', 'XRAY', 'DVN', 'FANG', 'DLR', 'DFS', 'DISCA', 'DISCK', 'DISH', 'DG', 'DLTR', 'D', 'DOV', 'DOW', 'DTE', 'DUK', 'DRE', 'DD', 'DXC', 'ETFC', 'EMN', 'ETN', 'EBAY', 'ECL', 'EIX', 'EW', 'EA', 'EMR', 'ETR', 'EOG', 'EFX', 'EQIX', 'EQR', 'ESS', 'EL', 'EVRG', 'ES', 'RE', 'EXC', 'EXPE', 'EXPD', 'EXR', 'XOM', 'FFIV', 'FB', 'FAST', 'FRT', 'FDX', 'FIS', 'FITB', 'FE', 'FRC', 'FISV', 'FLT', 'FLIR', 'FLS', 'FMC', 'F', 'FTNT', 'FTV', 'FBHS', 'FOXA', 'FOX', 'BEN', 'FCX', 'GPS', 'GRMN', 'IT', 'GD', 'GE', 'GIS', 'GM', 'GPC', 'GILD', 'GL', 'GPN', 'GS', 'GWW', 'HAL', 'HBI', 'HOG', 'HIG', 'HAS', 'HCA', 'HCP', 'HP', 'HSIC', 'HSY', 'HES', 'HPE', 'HLT', 'HFC', 'HOLX', 'HD', 'HON', 'HRL', 'HST', 'HPQ', 'HUM', 'HBAN', 'HII', 'IEX', 'IDXX', 'INFO', 'ITW', 'ILMN', 'IR', 'INTC', 'ICE', 'IBM', 'INCY', 'IP', 'IPG', 'IFF', 'INTU', 'ISRG', 'IVZ', 'IPGP', 'IQV', 'IRM', 'JKHY', 'JEC', 'JBHT', 'JEF', 'SJM', 'JNJ', 'JCI', 'JPM', 'JNPR', 'KSU', 'K', 'KEY', 'KEYS', 'KMB', 'KIM', 'KMI', 'KLAC', 'KSS', 'KHC', 'KR', 'LB', 'LHX', 'LH', 'LRCX', 'LW', 'LEG', 'LDOS', 'LEN', 'LLY', 'LNC',
'LIN', 'LKQ', 'LMT', 'L', 'LOW', 'LYB', 'MTB', 'MAC', 'M', 'MRO', 'MPC', 'MKTX', 'MAR', 'MMC', 'MLM', 'MAS', 'MA', 'MKC', 'MXIM', 'MCD', 'MCK', 'MDT', 'MRK', 'MET', 'MTD', 'MGM', 'MCHP', 'MU', 'MSFT', 'MAA', 'MHK', 'TAP', 'MDLZ', 'MNST', 'MCO', 'MS', 'MOS', 'MSI', 'MSCI', 'MYL', 'NDAQ', 'NOV', 'NKTR', 'NTAP', 'NFLX', 'NWL', 'NEM', 'NWSA', 'NWS', 'NEE', 'NLSN', 'NKE', 'NI', 'NBL', 'JWN', 'NSC', 'NTRS', 'NOC', 'NCLH', 'NRG', 'NUE', 'NVDA', 'ORLY', 'OXY', 'OMC', 'OKE', 'ORCL', 'PCAR', 'PKG', 'PH', 'PAYX', 'PYPL', 'PNR', 'PBCT',
'PEP', 'PKI', 'PRGO', 'PFE', 'PM', 'PSX', 'PNW', 'PXD', 'PNC', 'PPG', 'PPL', 'PFG', 'PG', 'PGR', 'PLD', 'PRU', 'PEG', 'PSA', 'PHM', 'PVH', 'QRVO', 'PWR', 'QCOM', 'DGX', 'RL', 'RJF', 'RTN', 'O', 'REG', 'REGN', 'RF', 'RSG', 'RMD', 'RHI', 'ROK', 'ROL', 'ROP', 'ROST', 'RCL', 'CRM', 'SBAC', 'SLB', 'STX', 'SEE', 'SRE', 'SHW', 'SPG', 'SWKS', 'SLG', 'SNA', 'SO', 'LUV', 'SPGI', 'SWK', 'SBUX', 'STT', 'SYK', 'STI', 'SIVB', 'SYMC', 'SYF', 'SNPS', 'SYY', 'TMUS', 'TROW', 'TTWO', 'TPR', 'TGT', 'TEL', 'FTI', 'TFX', 'TXN', 'TXT', 'TMO', 'TIF', 'TWTR', 'TJX', 'TSS', 'TSCO', 'TDG', 'TRV', 'TRIP', 'TSN', 'UDR', 'ULTA', 'USB', 'UAA', 'UA', 'UNP', 'UAL', 'UNH', 'UPS', 'URI', 'UTX', 'UHS', 'UNM', 'VFC', 'VLO', 'VAR', 'VTR', 'VRSN', 'VRSK', 'VZ', 'VRTX', 'VIAB', 'V', 'VNO', 'VMC', 'WAB', 'WMT', 'WBA', 'DIS', 'WM', 'WAT', 'WEC', 'WCG', 'WFC', 'WELL', 'WDC', 'WU', 'WRK', 'WY', 'WHR', 'WMB', 'WLTW', 'WYNN', 'XEL', 'XRX', 'XLNX', 'XYL', 'YUM', 'ZBH', 'ZION', 'ZTS']
MMM
ABT
ABBV
ABMD
ACN
ATVI
ADBE
AMD
AAP
AES
AMG
AFL
A
APD
AKAM
ALK
ALB
ARE
ALXN
ALGN
ALLE
AGN
ADS
LNT
ALL
GOOGL
GOOG
MO
AMZN
AMCR
AEE
AAL
AEP
AXP
AIG
AMT
AWK
AMP
ABC
AME
AMGN
APH
ADI
ANSS
ANTM
AON
AOS
APA
AIV
AAPL
AMAT
APTV
ADM
ARNC
ANET
AJG
AIZ
ATO
T
ADSK
ADP
AZO
AVB
AVY
BHGE
BLL
BAC
BK
BAX
BBT
BDX
BRK.B
Dot removed from variable - BRKB
BBY
BIIB
BLK
HRB
BA
BKNG
BWA
BXP
BSX
BMY
AVGO
BR
BF.B
Dot removed from variable - BFB
Traceback (most recent call last):
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pandas\core\indexes\base.py", line 2897, in get_loc
    return self._engine.get_loc(key)
  File "pandas\_libs\index.pyx", line 107, in pandas._libs.index.IndexEngine.get_loc
  File "pandas\_libs\index.pyx", line 131, in pandas._libs.index.IndexEngine.get_loc
  File "pandas\_libs\hashtable_class_helper.pxi", line 1607, in pandas._libs.hashtable.PyObjectHashTable.get_item
  File "pandas\_libs\hashtable_class_helper.pxi", line 1614, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 'Date'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "main.py", line 60, in <module>
    get_data_from_yahoo(True)
  File "main.py", line 55, in get_data_from_yahoo
    df = web.DataReader(ticker, 'yahoo', start, end)
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pandas_datareader\data.py", line 310, in DataReader
    session=session).read()
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pandas_datareader\base.py", line 210, in read
    params=self._get_params(self.symbols))
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pandas_datareader\yahoo\daily.py", line 142, in _read_one_data
    to_datetime(prices['Date'], unit='s').dt.date)
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pandas\core\frame.py", line 2980, in __getitem__
    indexer = self.columns.get_loc(key)
  File "C:\Program Files (x86)\Python37-32\lib\site-packages\pandas\core\indexes\base.py", line 2899, in get_loc
    return self._engine.get_loc(self._maybe_cast_indexer(key))
  File "pandas\_libs\index.pyx", line 107, in pandas._libs.index.IndexEngine.get_loc
  File "pandas\_libs\index.pyx", line 131, in pandas._libs.index.IndexEngine.get_loc
  File "pandas\_libs\hashtable_class_helper.pxi", line 1607, in pandas._libs.hashtable.PyObjectHashTable.get_item
  File "pandas\_libs\hashtable_class_helper.pxi", line 1614, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 'Date'

Code: Alles auswählen

import pandas as pd

SP500_URL = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'

def parse_symbols(url=SP500_URL):
    df = pd.read_html(url)[0]
    return df.Symbol.str.replace('.', '')

print(parse_symbols().head())