californian californian - 4 months ago 9
Python Question

HTTP client asynch calls with delay

I'm using httpclient.HTTPRequest library to send Async requests, but need to add delay between requests.
This means lets say I configure RPS (Requests per second) = 5. Then I send a request each 0.2 but asynchronously. How can I send the requests asynchronously without waiting for each request response.

This is my code:

def process_campaign(self, campaign_instance):
ioloop.IOLoop.current().run_sync(lambda: start_campaign(campaign_instance))

@gen.coroutine
def start_campaign(campaign_instance):
...
while True:
try:
log.info("start_campaign() Requests in Queue: {}".format(len(web_requests)))
web_request = web_requests.pop()
time.sleep(delay)
headers = {'Content-Type': 'application/json'}
request = httpclient.HTTPRequest(auth_username=settings.api_account,
auth_password=settings.api_password,
url=settings.api_url,
body=json.dumps(web_request),
headers=headers,
request_timeout=15,
method="POST")
response = yield http_client.fetch(request)

except httpclient.HTTPError, e:
log.exception("start_campaign() " + str(e))

except IndexError:
log.info('start_campaign() Campaign web requests completed. Errors {}'.format(api_errors))
break


But seems to wait for HTTP response before proceeding.

Answer

You can try:

class WebRequest(RequestHandler):
    def __init__(self, web_request):
        self.delay = 0
        self.web_request = web_request

    @asynchronous
    def post(self):
        IOLoop.instance().add_timeout(self.delay, self._process)

    @gen.coroutine
    def _process(self):
        try:
            http_client = httpclient.AsyncHTTPClient()
            log.info("start_campaign() Web request: {}".format(self.web_request))
            headers = {'Content-Type': 'application/json'}
            request = httpclient.HTTPRequest(auth_username=settings.api_account,
                                             auth_password=settings.api_password,
                                             url=settings.api_url,
                                             body=json.dumps(self.web_request),
                                             headers=headers,
                                             request_timeout=15,
                                             method="POST")
            response = yield http_client.fetch(request)
        except Exception, exception:
            log.exception(exception)

Re-use your while Loop:

while True:
            try:
                web_request = web_requests.pop()
                time.sleep(delay)
                client = WebRequest(web_request)
                client.post()

            except IndexError:               
                break
Comments