Mikail Mikail - 4 months ago 25
Node.js Question

node js request promise, scraping

I'm scraping a website , i use request library with node js, so I got an array of urls I loop through and do a request on each, the problem is that requests are asynchronous and I need to do something ONLY after all the requests are completed here is the code:

for (var i = 0; i < urls.length; i++) {

request(urls[i], function (err,resp,body) {

if (!err && resp.statusCode == 200) {
var $ = cheerio.load(body);
var string = $('.author .mini').text();
var regExp = /(\+971|00971|05)\d{1,12}/g;
if(string.match(regExp)) {mobilePhones.push(string.match(regExp)[0])}



}



});
}


so when all the requests are done, I just want to console.log(mobilePhones);

Answer
var count = urls.length;
for (var i = 0; i < urls.length; i++) {

    request(urls[i], function (err,resp,body) {

        if (!err && resp.statusCode == 200) {
            var $ = cheerio.load(body);
            var string = $('.author .mini').text();
            var regExp = /(\+971|00971|05)\d{1,12}/g;
            if(string.match(regExp)) {mobilePhones.push(string.match(regExp)[0])}
        }

        count--;
        if (count === 0) doSomething() // all requests are done.

    });
}