Das deutsche Python-Forum

Code: Alles auswählen

#!/usr/bin/env python3
import csv
import re
import time
from pathlib import Path
from shutil import copyfileobj
from urllib.parse import urljoin

import bs4
import requests
from loguru import logger

DOWNLOADS_PATH = Path.home() / "Downloads"
CSV_FILE_PATH = DOWNLOADS_PATH / "Free+English+textbooks.csv"
DELAY_BETWEEN_BOOKS = 5  # in seconds.


def process_book(url, download_path):
    response = requests.get(url)
    if not response.ok:
        logger.error(
            "Can not load book page: {} {}",
            response.status_code,
            response.reason,
        )
    else:
        soup = bs4.BeautifulSoup(response.text, "lxml")
        button_div_node = soup.find(
            "div", "main-body__content"
        ).find_next_sibling("div")
        if not button_div_node:
            logger.error("No buttons found.")
        else:
            download_urls = [
                urljoin(url, node["href"])
                for node in button_div_node("a", "c-button")
            ]
            if not download_urls:
                logger.warning("No download urls found!")
            else:
                logger.info(
                    "Found {} url(s): {}", len(download_urls), download_urls
                )
                for download_url in download_urls:
                    logger.info("Downloading {}", download_url)
                    response = requests.get(download_url, stream=True)
                    if not response.ok:
                        logger.error(
                            "Can not download book: {} {}",
                            response.status_code,
                            response.reason,
                        )
                    else:
                        #
                        # This is an ugly hack!
                        #
                        filename = re.sub(
                            r"^filename\*?=",
                            "",
                            response.headers["Content-Disposition"],
                            re.IGNORECASE,
                        )
                        logger.info("Saving as {!r}", filename)
                        with (download_path / filename).open(
                            "wb"
                        ) as book_file:
                            copyfileobj(response.raw, book_file, 1024 * 1024)
                logger.success("Processed {}", url)


@logger.catch
def main():
    logger.add("test.log")
    download_path = CSV_FILE_PATH.with_suffix("")
    download_path.mkdir(exist_ok=True)
    with CSV_FILE_PATH.open(encoding="utf-8") as csv_file:
        rows = csv.reader(csv_file)
        headers = next(rows)
        open_url_index = headers.index("OpenURL")
        for row_number, row in enumerate(rows, 1):
            url = row[open_url_index]
            logger.info("Processing {}: {}", row_number, url)
            process_book(url, download_path)
            time.sleep(DELAY_BETWEEN_BOOKS)


if __name__ == "__main__":
    main()

Code: Alles auswählen

import csv
import os
from pathlib import Path
import shutil

import requests
from bs4 import BeautifulSoup

csv_files = [file for file in os.listdir() if file.endswith(".csv")]

def webpage2soup(url):
    webpage = requests.get(url).text
    soup = BeautifulSoup(webpage, "html5lib")
    return soup

def get_url(soup):
    div_box = soup.find("div", attrs={"class": "cta-button-container__item"})
    a = 'https://link.springer.com' + '/' + div_box.find("a")["href"]
    return a
                    
    
def get_title(soup):
    title = soup.find("div", attrs={"class": "page-title"}).find("h1").text.replace("/", "+").replace(": ", " - ")
    return title

for csv_file in csv_files:
    os.mkdir(csv_file.replace(".csv",""))

for csv_file in csv_files:
    # Wir laden die csv_files und extrahieren hier die doi.org-URLs
    with open(csv_file, "r", encoding="utf-8") as f:
        bookreader = csv.reader(f, delimiter=';', quotechar='"')
        for row in bookreader:
            if row[17] != 'DOI URL':
                print(row[17])
                url = row[17]

                soup = webpage2soup(url)

                title = get_title(soup)

                pdf_url = get_url(soup)                           

                response = requests.get(pdf_url, stream=True)

                with open(Path(csv_file.replace(".csv", "")) / Path(title + ".pdf"), 'wb') as out_file:
                    shutil.copyfileobj(response.raw, out_file)