Das deutsche Python-Forum

Code: Alles auswählen

#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import json
import sys
import time
import urllib2
from contextlib import closing
from itertools import islice
from subprocess import PIPE, Popen
from threading import Thread

import MySQLdb


def fake_viewers(proxy_url):
    output = Popen(
        ['livestreamer', 'twitch.tv/shingshady', '-j'], stdout=PIPE
    ).communicate()[0]
    url = json.loads(output)['streams']['worst']['url']
    # 
    # BUG: Setting a global opener from a thread is of course not thread safe
    #   and leads to *all* threads requesting through the last set
    #   proxy handler.
    # 
    proxy_handler = urllib2.ProxyHandler({'http': proxy_url})
    opener = urllib2.build_opener(proxy_handler)
    urllib2.install_opener(opener)

    headers = {
        'User-Agent':
            'Mozilla/5.0 (Windows NT 6.1)'
            ' AppleWebKit/537.36 (KHTML, like Gecko)'
            ' Chrome/36.0.1985.143'
            ' Safari/537.36'
            ' OPR/23.0.1522.77'
    }
    while True:
        request = urllib2.Request(url, None, headers)
        request.get_method = lambda: 'HEAD'  # TODO: Use `requests` instead.
        # 
        # TODO: Limit the number of allowed connection errors per proxy.
        # 
        try:
            _response = urllib2.urlopen(request)
        except urllib2.HTTPError, error:
            print('HTTP ERROR:', error.code)
            print('-------', proxy_url, 'died! -----')
        except urllib2.URLError, error:
            print('URL ERROR:', error.reason)
            print('-------', proxy_url, 'died! -----')

        time.sleep(0.3)
    

def main():
    thread_count = int(sys.argv[1])

    with closing(MySQLdb.connect(host='xxxxxxxxx')) as database:
        cursor = database.cursor()
        cursor.execute('SELECT proxy FROM proxylist WHERE isStable = 1')
        proxy_urls = [row[0] for row in cursor.fetchall()]
 
    for proxy_url in islice(proxy_urls, thread_count):
        thread = Thread(target=fake_viewers, args=(proxy_url,))
        thread.start()
        print('Viewer Thread (', proxy_url, ') started')


if __name__ == '__main__':
    main()

Code: Alles auswählen

    http_proxy = "http://" + proxy_url

    proxyDict = {
        "http" : http_proxy
    }

    headers = {
        'User-Agent':
            'Mozilla/5.0 (Windows NT 6.1)'
            ' AppleWebKit/537.36 (KHTML, like Gecko)'
            ' Chrome/36.0.1985.143'
            ' Safari/537.36'
            ' OPR/23.0.1522.77'
    }

    while True:
        #
        # TODO: Limit the number of allowed connection errors per proxy.
        #
        try:
            response = requests.head(url, proxies=proxyDict,  headers=headers)
        except requests.exceptions.Timeout:
            print('-------', proxy_url, 'timed out! -----')
        except requests.exceptions.TooManyRedirects:
            print('-------', proxy_url, 'too many redirects on destination! -----')
        except requests.exceptions.RequestException as error:
            print('-------', error)
        time.sleep(0.3)

Code: Alles auswählen

#!/usr/bin/env python
from __future__ import absolute_import, division, print_function
import json
import sys
import time
import requests
from Queue import Queue
from contextlib import closing
from subprocess import PIPE, Popen
from threading import Thread

import MySQLdb

def get_url():
    output = Popen(
        ['livestreamer', 'twitch.tv/mag1x', '-j'], stdout=PIPE
    ).communicate()[0]
    return json.loads(output)['streams']['worst']['url']


def fake_viewers():
    error_counter = 0
    isProxySet = 0

    url = url_queue.get()
    url_queue.task_done()

    while True:
        if isProxySet == 0:
            proxy_url = proxy_queue.get()
            print('Proxy URL', proxy_url)
            print('Twitch URL', url)

            http_proxy = "http://" + proxy_url

            proxyDict = {
                "http" : http_proxy
            }

            headers = {
                'User-Agent':
                    'Mozilla/5.0 (Windows NT 6.1)'
                    ' AppleWebKit/537.36 (KHTML, like Gecko)'
                    ' Chrome/36.0.1985.143'
                    ' Safari/537.36'
                    ' OPR/23.0.1522.77'
            }

        isProxySet = 1

        try:
            response = requests.head(url, proxies=proxyDict,  headers=headers)
        except:
            print('-------', proxy_url, 'ERROR -----')
            error_counter += 1

#        except requests.exceptions.Timeout:
#            print('-------', proxy_url, 'timed out! -----')
#            error_counter += 1
#        except requests.exceptions.TooManyRedirects:
#            error_counter += 1
#            print('-------', proxy_url, 'too many redirects on destination! -----')
#        except requests.exceptions.RequestException as error:
#            error_counter += 1
#            print('-------', error)

        if error_counter == 5:
            error_counter = 0
            isProxySet = 0
            proxy_queue.task_done()

        time.sleep(0.3)

def main():
    thread_count = int(sys.argv[1])

    with closing(MySQLdb.connect(host="localhost", xxxxxxxxxx)) as database:
        cursor = database.cursor()
        cursor.execute('SELECT proxy FROM proxylist WHERE isStable = 1')
        proxy_urls = [row[0] for row in cursor.fetchall()]

    for proxy_url in proxy_urls:
        proxy_queue.put(proxy_url)

    for i in range(0, thread_count):
        url = get_url()
        url_queue.put(url)
        thread = Thread(target=fake_viewers)
      thread.start()
        print('Viewer Thread (', i , ') started')
        url_queue.join()

if __name__ == '__main__':
    proxy_queue = Queue()
    url_queue = Queue()
    main()