Cookbook

Automatic Sleep-Retries (used for throttling)

def make_throttle_hook(timeout=2):
    """Returns a response hook function which sleeps for <timeout> seconds if
       the response status_code/content meet certain criteria before
       re-requesting the content. The timeout length grows for each re-request
       until it exceeds a threshold, at which point simply return the response object.
    """

    import time 
    def myhook(resp):
        if timeout < 1025:
            if (resp.status_code == 403) and ("too_many" in resp.content):
                time.sleep(timeout)
                proxies = resp.request.proxies
                cookies = resp.request.cookies
                auth = resp.request.auth
                hooks = {'response': make_throttle_hook(timeout=timeout*3)}
                return requests.get(resp.url, proxies=proxies, cookies=cookies, auth=auth, hooks=hooks)
        return resp
    return myhook

resp = requests.get(url, hooks={'response': make_throttle_hook(5)})