Juntae Juntae - 4 months ago 29
Node.js Question

Function iteration with Async waterfall in Node.js

I am trying to get resource from the some url, that resource is separated as multiple pages, the number of page is over 500, but the order of resource have to be guaranteed, so I decided to use Async module.

function getData(page, callback){
var url = "http://someurl.com/document?page="+page;
// get data from the url ...
// when success,
callback();
};


So, above function is get resource from some url, I have to iterate this function to many times, but I don't know how can I iterate this with async waterfall. What point I should push the for iteration?

async.waterfall([

// page 1
function(callback){
getData(1, function(){
callback(null);
});
},

// page 2
function(callback){
getData(2, function(){
callback(null);
});
},

// page 3, 4, 5..... 100, ... 500


],function(err, result){
if(err) return next();

console.log('finish !');
});

Answer

Why don't you use Promises:

function getData(page, callback){
    var url = "http://someurl.com/document?page="+page;
    // var request = call to get data from the url ... 
    return request;
};

var promises = [];

for (var page = 1; page < totalPages; page++) {
    promises.push(getData(page));
}

Promise.all(promises).then(function (listOfResults) {
    // Do callback here
});

Just make sure that your method of sending the get request returns a promise. If not you can use a function like this:

function usesCallback(data, callback) {
    // Do something with data
    callback();
}

function makePromise(data) {
    return new Promise(function (resolve, reject) {
        usesCallback(data, resolve);
    });
}

Here is some more info on Promises

Comments