Das deutsche Python-Forum

Code: Alles auswählen

import os
import pathlib
import re
import time
import urllib.request
from datetime import datetime

from bs4 import BeautifulSoup
from pysitemap import crawler

start = f'https://www.gentool.net/data/zh/'
download_path = "./downloads"


def getLinks(start_url):
    print(f"Start-URL: {start_url}")
    path = pathlib.PurePath(start_url)
    print(path.parts[(len(path.parts) - 1)])

    str_path = path.parts[(len(path.parts) - 1)]
    html_page = urllib.request.urlopen(start_url)
    soup = BeautifulSoup(html_page, "lxml")
    links = []

    for link in soup.findAll('a'):
        regex = re.compile(r'\?|data')
        if not regex.search(link.get('href')):
            print(f"Link: {link.get('href')}")

            if not str_path in "zh":
                create_path = f"{download_path}/{str_path}/{link.get('href')}"
                print(f"Create Path: {create_path}")
            else:
                create_path = f"{download_path}/{link.get('href')}"
                print("no path")
            links.append(link.get('href'))

            if not os.path.exists(create_path):
                os.makedirs(create_path)
    return links


def cleanup_links(filename):
    with open(filename, "r+") as f:
        d = f.readlines()
        f.seek(0)
        for idx, i in enumerate(d):
            if "txt" in i:
                print(idx, i)
                f.write(i)
        f.truncate()

        if os.stat(filename).st_size == 0:
            print(" Removing ", filename)
            os.remove(filename)


if __name__ == '__main__':
    while True:
        months = getLinks(start)
        for month in months:
            if month == "2022_11_November/":
                filename = f'{month.replace("/", "")}.txt'
                print(start + month)
                # getLinks(start + month)
                crawler(start + month, out_file=filename, exclude_urls=[".rep", ".jpg", "?"], out_format="txt")
        cleanup_links("2022_11_November.txt")
        print(f"Sleeping 30min {datetime.now()}")
        time.sleep(900)

Code: Alles auswählen

def cleanup_links(filename):
    with open(filename, encoding="ascii") as links:
        cleaned_links = [
            link for link in links
            if "txt" in link
        ]
    if cleaned_links:
        with open(filename, "w", encoding="ascii") as links:
            links.writelines(cleaned_links)
    else:
        os.remove(filename)