Das deutsche Python-Forum

Code: Alles auswählen

#!/usr/bin/python

import mechanize
import urllib2
import re
from bs4 import BeautifulSoup

# I am doing a lot of not so plausible stuff, i better pretend to be a browser
browser = mechanize.Browser()
#cookie jar
cookies = cookielib.MozillaCookieJar('cookie_jar')
# set up the emulated browser
browser.set_cookiejar(cookies)
browser.set_handle_redirect(True)
browser.set_handle_robots(False)
browser.set_handle_equiv(True)
browser.set_handle_gzip(False)
browser.set_handle_referer(True)
browser.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
browser.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:14.0) Gecko/20120405 Firefox/14.0a1')]

# prep my lists
urls = []
wordlist = []

#prep containers
alltextinsoup = open('wordlist.txt', 'w')
urlsfile = open('links.txt', 'w')

# cook the initial soup
target = raw_input("enter target to scrape for words_> ")
r = browser.open("http://%s"%target)
html = r.read()
soup = BeautifulSoup(html)

# find all links in initial soup
for eachlink in soup.findAll('a'):
	urls.append(eachlink.get('href'))

for link in urls:
        # cook a soup for every links content
	r = browser.open(link)
	html = r.read()
	soup = BeautifulSoup(html)
	rawtext = (soup.get_text()).encode('utf-8', 'ignore')
        # i just want the clean words
	rawtext = re.sub('[^A-Za-z ]+', '', rawtext)
	splittedrawtext = rawtext.split(' ')
	for x in splittedrawtext:
		if x not in wordlist and (len(x) > 3 and len(x) < 12):
			wordlist.append(x.encode('utf-8'))
			for item in wordlist:
				alltextinsoup.write(str(item) + '\n')	
	
################################	
alltextinsoup.close()

Code: Alles auswählen

url=urlparse.urljoin(absurl,relurl)

Code: Alles auswählen

re.findall(r"(?:\s|^)(\w+)(?=\s|$)", rawtext)

Code: Alles auswählen

for eachlink in soup.findAll('a', href=True):
    eachlink['href'] = urlparse.urljoin(target, eachlink['href'])
    if eachlink not in linklist:
        linklist.append(eachlink['href'])