Kulin Kulin - 7 days ago 5
Javascript Question

Loop GET requests with incremental parameters

I am trying to GET data, from a lot of pages (same page, but different data). The problem, I think is with the async request method of Node, and the 'for' loop. (I'm new in nodejs).

I use a for loop to get JSON data using a request, but the loop was finishes before I get the first response of request (I think).

This is the code, for every day I get on JSON:

var str = [];
for (var i = 0; i < 2000; i++) {

var url = 'https://WEbPAGE.com/public?start='+prevDate+'&end='+nextDate;

request(url, (error, response, body)=> {
if (!error && response.statusCode === 200) {

str.push(body+'\r\n')

}
})

prevDate = nextDate;
nextDate += oneDay;

if (nextDate >= today) break;
};


When finalized, I get all of data from requests, but not ordered, and a lot of positions of array are empty.

Answer

Use a function with a callback instead of a for loop, keep running the same function as long as the dates don't match, then once date = today, callback.

function addData(prevdate,nextDate,cb){
    var url = 'https://WEbPAGE.com/public?start='+prevDate+'&end='+nextDate;
    request(url, (error, response, body)=> {
        if (!error && response.statusCode === 200) {
            str = str.concat(body+'\r\n')
            prevDate = nextDate;
            nextDate += oneDay; 
            if (nextDate >= today){
                cb()
            }else{
                addData(prevdate,nextDate,cb)
            }
        }
    })
}

addData(firstPrevDate,firstNextDate,function(){
     // str is ready!
});

Edit: As pointed out in the comments, requests in this example will be executed serially. you could modify the approach to run them a few at a time, but you should probably not try running 2000 parallel requests, or you're gonna have a bad time!