bei dem Parser bricht das Auswerten der Daten nach 20 Records ab.
Was ist beabsichtigt: ... die Pages der Volunteering-Angebote zu sammeln u. parsen und als CSV zu speichern.
m.a.W:
- https://europa.eu/youth/volunteering/organisation/50188
- https://europa.eu/youth/volunteering/organisation/50190
und so weiter und so fort: es gibt mor als 6000 records - doch der Parser bricht nach 20 Zeilen ab: m.a.W. es gibt eine CSV-Ausgabe mit nur 20 Zeilen...
der Zugang :
Code: Alles auswählen
import requests
from bs4 import BeautifulSoup
import re
import csv
from tqdm import tqdm
first = "https://europa.eu/youth/volunteering/organisations_en?page={}"
second = "https://europa.eu/youth/volunteering/organisation/{}_en"
def catch(url):
with requests.Session() as req:
pages = []
print("Loading All IDS\n")
for item in tqdm(range(0, 347)):
r = req.get(url.format(item))
soup = BeautifulSoup(r.content, 'html.parser')
numbers = [item.get("href").split("/")[-1].split("_")[0] for item in soup.findAll(
"a", href=re.compile("^/youth/volunteering/organisation/"), class_="btn btn-default")]
pages.append(numbers)
return numbers
def parse(url):
links = catch(first)
with requests.Session() as req:
with open("Data.csv", 'w', newline="", encoding="UTF-8") as f:
writer = csv.writer(f)
writer.writerow(["Name", "Address", "Site", "Phone",
"Description", "Scope", "Rec", "Send", "PIC", "OID", "Topic"])
print("\nParsing Now... \n")
for link in tqdm(links):
r = req.get(url.format(link))
soup = BeautifulSoup(r.content, 'html.parser')
task = soup.find("section", class_="col-sm-12").contents
name = task[1].text
add = task[3].find(
"i", class_="fa fa-location-arrow fa-lg").parent.text.strip()
try:
site = task[3].find("a", class_="link-default").get("href")
except:
site = "N/A"
try:
phone = task[3].find(
"i", class_="fa fa-phone").next_element.strip()
except:
phone = "N/A"
desc = task[3].find(
"h3", class_="eyp-project-heading underline").find_next("p").text
scope = task[3].findAll("span", class_="pull-right")[1].text
rec = task[3].select("tbody td")[1].text
send = task[3].select("tbody td")[-1].text
pic = task[3].select(
"span.vertical-space")[0].text.split(" ")[1]
oid = task[3].select(
"span.vertical-space")[-1].text.split(" ")[1]
topic = [item.next_element.strip() for item in task[3].select(
"i.fa.fa-check.fa-lg")]
writer.writerow([name, add, site, phone, desc,
scope, rec, send, pic, oid, "".join(topic)])
parse(second)
Ich denke dass ich am Ende der ersten Funktion auch eher Pages als numbers zurückgeben sollte.
Code: Alles auswählen
pages.append(numbers)
return numbers
Was meint Ihr denn!?