Das deutsche Python-Forum

Code: Alles auswählen

from socket import *
import urllib2
import re


se = [
	{
	"host": "www.google.de",
	"path": "/search",
	"keyword": "q",
	"regex": re.compile('<a class=l href=".+">'),
	"pageName": "start",
	"pageAdd": 100,
	"pageInit":	0,
	"aufJeden": {"num": 100, "filter": "off"}
	},
	{
	"host": "search.live.com", #immer nur 10 ergebnisse auf einmal, mit cookies waeren 50 moeglich
	"path": "/results.aspx",
	"keyword": "q",
	#"regex": re.compile(t4ht),
	"pageName": "first",
	"pageAdd": 50,
	"pageInit":	1,
	"aufJeden": {}
	}
	]

	
def mkKeyword():
	return "lol"

class Search:
	def __init__(self, ARG):
		self.ARG = ARG
		self.not_urls = 0
		self.current_page = self.ARG["pageInit"]
		self.keyword = "fd"
	
	def _dict2cgi(self, dict):
		retval = "?"
		for key, val in dict.iteritems():
			retval += key + "=" + str(val) + "&"
		return retval[:-1]
	
	def _get(self, arg):#nein, kein urllib2
		arg = self._dict2cgi(arg)
		s = socket(AF_INET, SOCK_STREAM)
		print "httpGetting..."
		s.connect((self.ARG["host"], 80))
		s.send("GET " + self.ARG["path"] + arg + " HTTP/1.1\n")
		s.send('Host: ' + self.ARG["host"])
		s.send('\n\n')
		resp = ''
		while not '</html>' in resp:
			resp += s.recv(1024)
		return resp
	
	def _parse(self, resp):
		print self.ARG["regex"]
		return self.ARG["regex"].findall(resp)
	
	def __call__(self):
		if self.not_urls == 1:			
			self.keyword = mkKeyword()
			current_page = self.ARG["pageInit"]
		elif self.not_urls == 2:
			return "ausgedient"
		
		z = self.ARG["aufJeden"]
		z.update({self.ARG["pageName"]: self.current_page, self.ARG["keyword"]: self.keyword})
		print z
		resp = self._get(z) #  + mit dicts?
		urls = self._parse(resp)
		if not urls:
			self.not_urls += 1
			print "keine urls"
		self.current_page += self.ARG["pageAdd"]
		return urls


print Search(se[0])()
"""
sObjs = []
for i in se:
	sObjs.append(Search(i))

while 1:
	for i in sObjs:
		print i()

"""

Code: Alles auswählen

import SimpleHTTPServer